I0311 03:22:24.408736 1 cmd.go:253] Using service-serving-cert provided certificates I0311 03:22:24.409134 1 observer_polling.go:159] Starting file observer W0311 03:22:24.428626 1 builder.go:272] unable to get owner reference (falling back to namespace): replicasets.apps "insights-operator-5ff5cb4f99" is forbidden: User "system:serviceaccount:openshift-insights:operator" cannot get resource "replicasets" in API group "apps" in the namespace "openshift-insights" I0311 03:22:24.960770 1 operator.go:59] Starting insights-operator v0.0.0-master+$Format:%H$ I0311 03:22:24.961090 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0311 03:22:24.962020 1 secure_serving.go:57] Forcing use of http/1.1 only W0311 03:22:24.962046 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0311 03:22:24.962054 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0311 03:22:24.962061 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0311 03:22:24.962066 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. I0311 03:22:24.962062 1 simple_featuregate_reader.go:171] Starting feature-gate-detector W0311 03:22:24.962071 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0311 03:22:24.962075 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0311 03:22:24.965697 1 event.go:377] Event(v1.ObjectReference{Kind:"Namespace", Namespace:"openshift-insights", Name:"openshift-insights", UID:"", APIVersion:"v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AWSEFSDriverVolumeMetrics", "AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BareMetalLoadBalancer", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ChunkSizeMiB", "CloudDualStackNodeIPs", "ConsolePluginContentSecurityPolicy", "DisableKubeletCloudCredentialProviders", "ExternalOIDC", "GCPLabelsTags", "GatewayAPI", "GatewayAPIController", "HardwareSpeed", "IngressControllerLBSubnetsAWS", "KMSv1", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "MultiArchInstallAWS", "MultiArchInstallGCP", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "NodeDisruptionPolicy", "OnClusterBuild", "PersistentIPsForVirtualization", "PrivateHostedZoneAWS", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "VSphereDriverConfiguration", "VSphereMultiVCenters", "ValidatingAdmissionPolicy"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AutomatedEtcdBackup", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "HighlyAvailableArbiter", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "InsightsRuntimeExtractor", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "MachineAPIProviderOpenStack", "MachineConfigNodes", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PinnedImages", "PlatformOperators", "ProcMountType", "RouteAdvertisements", "SELinuxChangePolicy", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass", "VolumeGroupSnapshot"}} I0311 03:22:24.965693 1 operator.go:124] FeatureGates initialized: knownFeatureGates=[AWSEFSDriverVolumeMetrics AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BareMetalLoadBalancer BuildCSIVolumes CPMSMachineNamePrefix ChunkSizeMiB CloudDualStackNodeIPs ConsolePluginContentSecurityPolicy DisableKubeletCloudCredentialProviders ExternalOIDC GCPLabelsTags GatewayAPI GatewayAPIController HardwareSpeed IngressControllerLBSubnetsAWS KMSv1 ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles MultiArchInstallAWS MultiArchInstallGCP NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation NodeDisruptionPolicy OnClusterBuild PersistentIPsForVirtualization PrivateHostedZoneAWS RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController VSphereDriverConfiguration VSphereMultiVCenters ValidatingAdmissionPolicy AWSClusterHostedDNS AutomatedEtcdBackup BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNS GCPCustomAPIEndpoints HighlyAvailableArbiter ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather InsightsRuntimeExtractor KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController MachineAPIProviderOpenStack MachineConfigNodes MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PinnedImages PlatformOperators ProcMountType RouteAdvertisements SELinuxChangePolicy SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerification SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass VolumeGroupSnapshot] I0311 03:22:24.965822 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0311 03:22:24.965840 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0311 03:22:24.965855 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0311 03:22:24.965910 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0311 03:22:24.966095 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0311 03:22:24.966280 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0311 03:22:24.966309 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0311 03:22:24.966716 1 secure_serving.go:213] Serving securely on [::]:8443 I0311 03:22:24.966753 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" E0311 03:22:24.968974 1 event.go:359] "Server rejected event (will not retry!)" err="events is forbidden: User \"system:serviceaccount:openshift-insights:operator\" cannot create resource \"events\" in API group \"\" in the namespace \"openshift-insights\"" event="&Event{ObjectMeta:{openshift-insights.189bab68d0a227e1 openshift-insights 0 0001-01-01 00:00:00 +0000 UTC map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Namespace,Namespace:openshift-insights,Name:openshift-insights,UID:,APIVersion:v1,ResourceVersion:,FieldPath:,},Reason:FeatureGatesInitialized,Message:FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{\"AWSEFSDriverVolumeMetrics\", \"AdditionalRoutingCapabilities\", \"AdminNetworkPolicy\", \"AlibabaPlatform\", \"AzureWorkloadIdentity\", \"BareMetalLoadBalancer\", \"BuildCSIVolumes\", \"CPMSMachineNamePrefix\", \"ChunkSizeMiB\", \"CloudDualStackNodeIPs\", \"ConsolePluginContentSecurityPolicy\", \"DisableKubeletCloudCredentialProviders\", \"ExternalOIDC\", \"GCPLabelsTags\", \"GatewayAPI\", \"GatewayAPIController\", \"HardwareSpeed\", \"IngressControllerLBSubnetsAWS\", \"KMSv1\", \"ManagedBootImages\", \"ManagedBootImagesAWS\", \"MetricsCollectionProfiles\", \"MultiArchInstallAWS\", \"MultiArchInstallGCP\", \"NetworkDiagnosticsConfig\", \"NetworkLiveMigration\", \"NetworkSegmentation\", \"NodeDisruptionPolicy\", \"OnClusterBuild\", \"PersistentIPsForVirtualization\", \"PrivateHostedZoneAWS\", \"RouteExternalCertificate\", \"ServiceAccountTokenNodeBinding\", \"SetEIPForNLBIngressController\", \"VSphereDriverConfiguration\", \"VSphereMultiVCenters\", \"ValidatingAdmissionPolicy\"}, Disabled:[]v1.FeatureGateName{\"AWSClusterHostedDNS\", \"AutomatedEtcdBackup\", \"BootcNodeManagement\", \"ClusterAPIInstall\", \"ClusterAPIInstallIBMCloud\", \"ClusterMonitoringConfig\", \"ClusterVersionOperatorConfiguration\", \"DNSNameResolver\", \"DualReplica\", \"DyanmicServiceEndpointIBMCloud\", \"DynamicResourceAllocation\", \"EtcdBackendQuota\", \"EventedPLEG\", \"Example\", \"Example2\", \"ExternalOIDCWithUIDAndExtraClaimMappings\", \"GCPClusterHostedDNS\", \"GCPCustomAPIEndpoints\", \"HighlyAvailableArbiter\", \"ImageStreamImportMode\", \"IngressControllerDynamicConfigurationManager\", \"InsightsConfig\", \"InsightsConfigAPI\", \"InsightsOnDemandDataGather\", \"InsightsRuntimeExtractor\", \"KMSEncryptionProvider\", \"MachineAPIMigration\", \"MachineAPIOperatorDisableMachineHealthCheckController\", \"MachineAPIProviderOpenStack\", \"MachineConfigNodes\", \"MaxUnavailableStatefulSet\", \"MinimumKubeletVersion\", \"MixedCPUsAllocation\", \"MultiArchInstallAzure\", \"NewOLM\", \"NewOLMCatalogdAPIV1Metas\", \"NewOLMOwnSingleNamespace\", \"NewOLMPreflightPermissionChecks\", \"NodeSwap\", \"NutanixMultiSubnets\", \"OVNObservability\", \"OpenShiftPodSecurityAdmission\", \"PinnedImages\", \"PlatformOperators\", \"ProcMountType\", \"RouteAdvertisements\", \"SELinuxChangePolicy\", \"SELinuxMount\", \"ShortCertRotation\", \"SignatureStores\", \"SigstoreImageVerification\", \"SigstoreImageVerificationPKI\", \"TranslateStreamCloseWebsocketRequests\", \"UpgradeStatus\", \"UserNamespacesPodSecurityStandards\", \"UserNamespacesSupport\", \"VSphereConfigurableMaxAllowedBlockVolumesPerNode\", \"VSphereHostVMGroupZonal\", \"VSphereMultiDisk\", \"VSphereMultiNetworks\", \"VolumeAttributesClass\", \"VolumeGroupSnapshot\"}},Source:EventSource{Component:openshift-insights-operator,Host:,},FirstTimestamp:2026-03-11 03:22:24.965658593 +0000 UTC m=+0.595009513,LastTimestamp:2026-03-11 03:22:24.965658593 +0000 UTC m=+0.595009513,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:openshift-insights-operator,ReportingInstance:,}" W0311 03:22:24.971196 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0311 03:22:24.971221 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0311 03:22:24.971295 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0311 03:22:24.975153 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0311 03:22:24.975171 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0311 03:22:24.978593 1 secretconfigobserver.go:119] support secret does not exist I0311 03:22:24.982302 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0311 03:22:24.985519 1 secretconfigobserver.go:119] support secret does not exist I0311 03:22:24.988129 1 recorder.go:156] Pruning old reports every 5h5m32s, max age is 288h0m0s I0311 03:22:24.993294 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0311 03:22:24.993305 1 periodic.go:212] Running clusterconfig gatherer I0311 03:22:24.993314 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0311 03:22:24.993339 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0311 03:22:24.993345 1 insightsreport.go:296] Starting report retriever I0311 03:22:24.993351 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0311 03:22:24.993355 1 tasks_processing.go:45] number of workers: 32 I0311 03:22:24.993380 1 tasks_processing.go:69] worker 1 listening for tasks. I0311 03:22:24.993390 1 tasks_processing.go:69] worker 0 listening for tasks. I0311 03:22:24.993394 1 tasks_processing.go:71] worker 0 working on silenced_alerts task. I0311 03:22:24.993397 1 tasks_processing.go:69] worker 8 listening for tasks. I0311 03:22:24.993404 1 tasks_processing.go:69] worker 4 listening for tasks. I0311 03:22:24.993404 1 tasks_processing.go:69] worker 9 listening for tasks. I0311 03:22:24.993410 1 tasks_processing.go:69] worker 10 listening for tasks. I0311 03:22:24.993416 1 tasks_processing.go:69] worker 5 listening for tasks. I0311 03:22:24.993417 1 tasks_processing.go:69] worker 11 listening for tasks. I0311 03:22:24.993423 1 tasks_processing.go:69] worker 3 listening for tasks. I0311 03:22:24.993424 1 tasks_processing.go:69] worker 22 listening for tasks. I0311 03:22:24.993430 1 tasks_processing.go:69] worker 6 listening for tasks. I0311 03:22:24.993432 1 tasks_processing.go:69] worker 23 listening for tasks. I0311 03:22:24.993430 1 tasks_processing.go:69] worker 2 listening for tasks. I0311 03:22:24.993439 1 tasks_processing.go:69] worker 16 listening for tasks. I0311 03:22:24.993438 1 tasks_processing.go:69] worker 21 listening for tasks. I0311 03:22:24.993438 1 tasks_processing.go:69] worker 7 listening for tasks. I0311 03:22:24.993442 1 tasks_processing.go:69] worker 24 listening for tasks. I0311 03:22:24.993449 1 tasks_processing.go:69] worker 25 listening for tasks. I0311 03:22:24.993448 1 tasks_processing.go:69] worker 14 listening for tasks. I0311 03:22:24.993450 1 tasks_processing.go:69] worker 13 listening for tasks. I0311 03:22:24.993455 1 tasks_processing.go:69] worker 15 listening for tasks. I0311 03:22:24.993458 1 tasks_processing.go:69] worker 26 listening for tasks. I0311 03:22:24.993452 1 tasks_processing.go:69] worker 17 listening for tasks. I0311 03:22:24.993463 1 tasks_processing.go:69] worker 19 listening for tasks. I0311 03:22:24.993459 1 tasks_processing.go:69] worker 12 listening for tasks. I0311 03:22:24.993467 1 tasks_processing.go:69] worker 20 listening for tasks. I0311 03:22:24.993470 1 tasks_processing.go:69] worker 29 listening for tasks. I0311 03:22:24.993471 1 tasks_processing.go:69] worker 30 listening for tasks. I0311 03:22:24.993468 1 tasks_processing.go:69] worker 27 listening for tasks. I0311 03:22:24.993474 1 tasks_processing.go:69] worker 31 listening for tasks. I0311 03:22:24.993477 1 tasks_processing.go:69] worker 28 listening for tasks. I0311 03:22:24.993459 1 tasks_processing.go:69] worker 18 listening for tasks. I0311 03:22:24.993486 1 tasks_processing.go:71] worker 4 working on operators_pods_and_events task. I0311 03:22:24.993482 1 tasks_processing.go:71] worker 8 working on monitoring_persistent_volumes task. I0311 03:22:24.993491 1 tasks_processing.go:71] worker 28 working on machine_autoscalers task. I0311 03:22:24.993493 1 tasks_processing.go:71] worker 18 working on proxies task. I0311 03:22:24.993529 1 tasks_processing.go:71] worker 9 working on pod_network_connectivity_checks task. I0311 03:22:24.993394 1 tasks_processing.go:71] worker 1 working on nodes task. I0311 03:22:24.993483 1 tasks_processing.go:71] worker 5 working on dvo_metrics task. I0311 03:22:24.993567 1 tasks_processing.go:71] worker 10 working on networks task. I0311 03:22:24.993690 1 tasks_processing.go:71] worker 11 working on version task. I0311 03:22:24.993990 1 tasks_processing.go:71] worker 14 working on storage_classes task. I0311 03:22:24.994000 1 tasks_processing.go:71] worker 3 working on openshift_logging task. I0311 03:22:24.994028 1 tasks_processing.go:71] worker 6 working on authentication task. I0311 03:22:24.994080 1 tasks_processing.go:71] worker 22 working on overlapping_namespace_uids task. I0311 03:22:24.994175 1 tasks_processing.go:71] worker 12 working on openshift_machine_api_events task. I0311 03:22:24.994221 1 tasks_processing.go:71] worker 7 working on operators task. I0311 03:22:24.994285 1 tasks_processing.go:71] worker 17 working on openstack_dataplanedeployments task. I0311 03:22:24.994322 1 tasks_processing.go:71] worker 21 working on tsdb_status task. I0311 03:22:24.994548 1 tasks_processing.go:71] worker 24 working on metrics task. I0311 03:22:24.994571 1 tasks_processing.go:71] worker 25 working on sap_pods task. I0311 03:22:24.993993 1 tasks_processing.go:71] worker 2 working on clusterroles task. I0311 03:22:24.994222 1 tasks_processing.go:71] worker 16 working on machines task. I0311 03:22:24.994188 1 tasks_processing.go:71] worker 30 working on ceph_cluster task. I0311 03:22:24.994183 1 tasks_processing.go:71] worker 20 working on nodenetworkstates task. I0311 03:22:24.994192 1 tasks_processing.go:71] worker 29 working on ingress_certificates task. I0311 03:22:24.994199 1 tasks_processing.go:71] worker 31 working on active_alerts task. I0311 03:22:24.994211 1 tasks_processing.go:71] worker 23 working on schedulers task. I0311 03:22:24.994265 1 tasks_processing.go:71] worker 26 working on install_plans task. I0311 03:22:24.994269 1 tasks_processing.go:71] worker 13 working on container_runtime_configs task. I0311 03:22:24.994273 1 tasks_processing.go:71] worker 15 working on jaegers task. I0311 03:22:24.994310 1 tasks_processing.go:71] worker 19 working on lokistack task. I0311 03:22:24.994196 1 tasks_processing.go:71] worker 27 working on machine_configs task. I0311 03:22:24.996304 1 tasks_processing.go:71] worker 28 working on cluster_apiserver task. I0311 03:22:24.996311 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 2.799392ms to process 0 records E0311 03:22:24.996322 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0311 03:22:24.996365 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 2.753869ms to process 0 records I0311 03:22:24.996394 1 tasks_processing.go:71] worker 9 working on image_registries task. E0311 03:22:24.998951 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: read udp 10.128.0.9:39785->172.30.0.10:53: read: connection refused E0311 03:22:24.998957 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: read udp 10.128.0.9:39785->172.30.0.10:53: read: connection refused I0311 03:22:24.998977 1 tasks_processing.go:71] worker 21 working on certificate_signing_requests task. E0311 03:22:24.998980 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: read udp 10.128.0.9:39785->172.30.0.10:53: read: connection refused I0311 03:22:24.998994 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 4.600534ms to process 0 records E0311 03:22:24.999006 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: read udp 10.128.0.9:39785->172.30.0.10:53: read: connection refused I0311 03:22:24.999024 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 4.387968ms to process 0 records I0311 03:22:24.999057 1 tasks_processing.go:71] worker 24 working on nodenetworkconfigurationpolicies task. E0311 03:22:24.999101 1 gather_active_alerts.go:64] Unable to retrieve most recent alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp: lookup alertmanager-main.openshift-monitoring.svc on 172.30.0.10:53: read udp 10.128.0.9:43559->172.30.0.10:53: read: connection refused I0311 03:22:24.999122 1 tasks_processing.go:71] worker 31 working on cost_management_metrics_configs task. E0311 03:22:24.999177 1 gather_silenced_alerts.go:52] Unable to retrieve silenced alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp: lookup alertmanager-main.openshift-monitoring.svc on 172.30.0.10:53: read udp 10.128.0.9:43559->172.30.0.10:53: read: connection refused E0311 03:22:24.999149 1 gather.go:140] gatherer "clusterconfig" function "active_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp: lookup alertmanager-main.openshift-monitoring.svc on 172.30.0.10:53: read udp 10.128.0.9:43559->172.30.0.10:53: read: connection refused I0311 03:22:24.999583 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 3.937804ms to process 0 records E0311 03:22:24.999603 1 gather.go:140] gatherer "clusterconfig" function "silenced_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp: lookup alertmanager-main.openshift-monitoring.svc on 172.30.0.10:53: read udp 10.128.0.9:43559->172.30.0.10:53: read: connection refused I0311 03:22:24.999618 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 5.794681ms to process 0 records I0311 03:22:24.999633 1 tasks_processing.go:71] worker 0 working on validating_webhook_configurations task. I0311 03:22:25.003026 1 tasks_processing.go:71] worker 20 working on storage_cluster task. I0311 03:22:25.003041 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 8.088705ms to process 0 records I0311 03:22:25.003327 1 tasks_processing.go:71] worker 10 working on service_accounts task. I0311 03:22:25.003738 1 recorder.go:70] Recording config/network with fingerprint=95bcc467b550e5977447a899f61d6723385716c64d0b13405f33ca1fab7b825e I0311 03:22:25.003793 1 gather.go:177] gatherer "clusterconfig" function "networks" took 9.752044ms to process 1 records I0311 03:22:25.003814 1 controller.go:119] Initializing last reported time to 0001-01-01T00:00:00Z I0311 03:22:25.003842 1 controller.go:317] The initial operator extension status is healthy I0311 03:22:25.003854 1 controller.go:203] Source periodic-workloads *controllerstatus.Simple is not ready I0311 03:22:25.003892 1 controller.go:203] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0311 03:22:25.003866 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 9.058165ms to process 0 records I0311 03:22:25.003903 1 controller.go:203] Source periodic-conditional *controllerstatus.Simple is not ready I0311 03:22:25.003919 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 8.541386ms to process 0 records I0311 03:22:25.003931 1 controller.go:458] The operator is still being initialized I0311 03:22:25.003937 1 tasks_processing.go:71] worker 25 working on mutating_webhook_configurations task. I0311 03:22:25.003931 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 8.800768ms to process 0 records E0311 03:22:25.003949 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0311 03:22:25.003960 1 gather.go:177] gatherer "clusterconfig" function "machines" took 8.729749ms to process 0 records I0311 03:22:25.003967 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 8.136509ms to process 0 records I0311 03:22:25.003975 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 8.062327ms to process 0 records I0311 03:22:25.003874 1 tasks_processing.go:71] worker 17 working on openstack_version task. I0311 03:22:25.003985 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 8.246101ms to process 0 records I0311 03:22:25.003993 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 9.736377ms to process 0 records I0311 03:22:25.003996 1 tasks_processing.go:71] worker 30 working on image_pruners task. I0311 03:22:25.004007 1 tasks_processing.go:71] worker 16 working on container_images task. I0311 03:22:25.004044 1 tasks_processing.go:71] worker 15 working on machine_config_pools task. I0311 03:22:25.004080 1 recorder.go:70] Recording config/proxy with fingerprint=a88dc0aae9e5708e7027c06e3d7729f3563477459c005f1dd2056ffcbd838f4e I0311 03:22:25.004137 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 10.397055ms to process 1 records I0311 03:22:25.003939 1 controller.go:481] The operator is healthy I0311 03:22:25.004325 1 tasks_processing.go:71] worker 3 working on pdbs task. I0311 03:22:25.004378 1 tasks_processing.go:71] worker 13 working on sap_config task. I0311 03:22:25.004341 1 tasks_processing.go:71] worker 18 working on machine_healthchecks task. I0311 03:22:25.004038 1 tasks_processing.go:71] worker 19 working on config_maps task. I0311 03:22:25.004673 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s I0311 03:22:25.004386 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0311 03:22:25.004612 1 recorder.go:70] Recording config/node/ip-10-0-0-47.ec2.internal with fingerprint=ee0d54b8b98b77aa5626466f7eeef2120fc527a4d8a1d3a3bc3a008a55ea3f6c I0311 03:22:25.004597 1 tasks_processing.go:71] worker 1 working on machine_sets task. W0311 03:22:25.005041 1 operator.go:287] started I0311 03:22:25.005060 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0311 03:22:25.005197 1 recorder.go:70] Recording config/node/ip-10-0-1-207.ec2.internal with fingerprint=8c6d7240f48b6712f94bed332298dddc10e984cbd50008cc325060ade9dc7edf I0311 03:22:25.005375 1 recorder.go:70] Recording config/node/ip-10-0-2-165.ec2.internal with fingerprint=78d5e888fc44db5a52c3bdff9d3b5047d75f65871fdc1052b8ea7579b3521ca9 I0311 03:22:25.005396 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 10.435713ms to process 3 records I0311 03:22:25.005495 1 tasks_processing.go:71] worker 6 working on infrastructures task. I0311 03:22:25.005603 1 recorder.go:70] Recording config/authentication with fingerprint=72c0e06512e8ef41398f31a1eeaf61ad6810e5b44c4b6e4606eda0e9114ce910 I0311 03:22:25.005618 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 10.12381ms to process 1 records W0311 03:22:25.007806 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0311 03:22:25.013529 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 14.392184ms to process 0 records I0311 03:22:25.013525 1 tasks_processing.go:71] worker 31 working on oauths task. I0311 03:22:25.022919 1 tasks_processing.go:71] worker 14 working on aggregated_monitoring_cr_names task. I0311 03:22:25.023676 1 recorder.go:70] Recording config/storage/storageclasses/gp2-csi with fingerprint=1a7ad9fbe51ef49927f77c7fd78bc94b1ffc6b475d66a5f741913c7d931ab499 I0311 03:22:25.023853 1 recorder.go:70] Recording config/storage/storageclasses/gp3-csi with fingerprint=488aa56e7994705a39e0b3419141a85cd55c2cf85ecb61989b796e36affbe1fa I0311 03:22:25.023891 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 28.913547ms to process 2 records I0311 03:22:25.023948 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 19.290416ms to process 0 records I0311 03:22:25.024531 1 tasks_processing.go:71] worker 15 working on node_logs task. I0311 03:22:25.024859 1 tasks_processing.go:71] worker 28 working on openstack_dataplanenodesets task. I0311 03:22:25.025273 1 recorder.go:70] Recording config/apiserver with fingerprint=2dd314f82c086ea1f4b5f06a6a1eafbb06f7060b0ecf0d970c56e811aaf3f7b2 I0311 03:22:25.025476 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 28.539445ms to process 1 records I0311 03:22:25.025531 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 26.075299ms to process 0 records I0311 03:22:25.025654 1 recorder.go:70] Recording config/schedulers/cluster with fingerprint=c293c0118af0e111a3fb03c6decee44261bf03b09102bf97e7e8d080bb3aae94 I0311 03:22:25.025692 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 30.104581ms to process 1 records I0311 03:22:25.025723 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 31.229531ms to process 0 records E0311 03:22:25.025738 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0311 03:22:25.025747 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 21.123463ms to process 0 records I0311 03:22:25.025695 1 tasks_processing.go:71] worker 24 working on crds task. I0311 03:22:25.025763 1 tasks_processing.go:71] worker 23 working on support_secret task. I0311 03:22:25.025836 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 32.261593ms to process 0 records I0311 03:22:25.025858 1 tasks_processing.go:71] worker 18 working on feature_gates task. I0311 03:22:25.025907 1 tasks_processing.go:71] worker 12 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0311 03:22:25.025757 1 tasks_processing.go:71] worker 8 working on ingress task. I0311 03:22:25.026282 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 21.165546ms to process 0 records I0311 03:22:25.026398 1 tasks_processing.go:71] worker 9 working on image task. I0311 03:22:25.026417 1 tasks_processing.go:71] worker 1 working on sap_datahubs task. I0311 03:22:25.027155 1 recorder.go:70] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=2536d3227fc02499af240a8e2aef82af5803f098a34a99829ee97ebb9b6e8ada I0311 03:22:25.027214 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 29.928007ms to process 1 records I0311 03:22:25.027283 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 23.933216ms to process 0 records E0311 03:22:25.027322 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0311 03:22:25.027375 1 recorder.go:70] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0311 03:22:25.027411 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 31.321386ms to process 1 records I0311 03:22:25.027438 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 23.024145ms to process 0 records I0311 03:22:25.027464 1 tasks_processing.go:71] worker 17 working on olm_operators task. I0311 03:22:25.027682 1 tasks_processing.go:74] worker 27 stopped. I0311 03:22:25.027714 1 recorder.go:70] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=deeefe95a14c83d57a437518435c7cb1430dc4caa78604f074ab9b8b2a0b15f3 I0311 03:22:25.027742 1 tasks_processing.go:74] worker 20 stopped. I0311 03:22:25.027729 1 tasks_processing.go:71] worker 30 working on openstack_controlplanes task. I0311 03:22:25.027785 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 23.586649ms to process 1 records I0311 03:22:25.027833 1 tasks_processing.go:74] worker 6 stopped. I0311 03:22:25.029506 1 recorder.go:70] Recording config/infrastructure with fingerprint=3b1bd73455fb609745cce8ffb50953e3a7d15a4c2a141f28f3fd3fa9c5c4f8a2 I0311 03:22:25.029558 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 22.32316ms to process 1 records I0311 03:22:25.029579 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 23.588154ms to process 0 records I0311 03:22:25.029608 1 tasks_processing.go:74] worker 3 stopped. I0311 03:22:25.029621 1 tasks_processing.go:74] worker 13 stopped. I0311 03:22:25.031266 1 controller.go:203] Source periodic-workloads *controllerstatus.Simple is not ready I0311 03:22:25.031327 1 controller.go:203] Source scaController *sca.Controller is not ready I0311 03:22:25.031352 1 recorder.go:70] Recording config/pdbs/openshift-console/console with fingerprint=8abaeb6aee19614ea8e089d0f29b2c46869d02302d533c6dbb6fa0e19d9af128 I0311 03:22:25.031428 1 recorder.go:70] Recording config/pdbs/openshift-console/downloads with fingerprint=48e00ab9aa946581ef101d656a36a9762986f0d05e375baa4c95c9ed11bdc8fc I0311 03:22:25.031495 1 recorder.go:70] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=9a4f9300b994d19d292155a91de0636bc13f0e34364afb2ddede78fce7101bde I0311 03:22:25.031549 1 recorder.go:70] Recording config/pdbs/openshift-ingress/router-default with fingerprint=2472ea81c2edac90ca1da188672b5ad378aea8ae282c535d6426e8abb8326277 I0311 03:22:25.031624 1 recorder.go:70] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=1e4148321730e0b4bf05a829d4b046a9c1e3c194453345aa9d668ae2dff6e6b1 I0311 03:22:25.031684 1 recorder.go:70] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=583612c71205bb32c6fd44bf067ec883cffa92cbd3564f4977e726168aaf643a I0311 03:22:25.031751 1 recorder.go:70] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=41c4627686f3e746dd614bee36194ec5b5e6091e045dc84f9a7a25b964afd0d9 I0311 03:22:25.031803 1 recorder.go:70] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=172e9f356ce3b78dfac7f356ae206226ddf7df5e520cdb8ce01c35b3dc270302 I0311 03:22:25.032486 1 recorder.go:70] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=e1e0eb2988d34a6d0785b462f84f6f7adfd551b639469c5c0377392af2660f40 I0311 03:22:25.032527 1 recorder.go:70] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=45244b933c160b284d1a3d2ed1b7fabac63220475ac938783e3bf535ca2971c2 I0311 03:22:25.032552 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 24.860283ms to process 10 records I0311 03:22:25.032579 1 recorder.go:70] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0311 03:22:25.033208 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 35.280074ms to process 1 records I0311 03:22:25.031361 1 controller.go:203] Source clusterTransferController *clustertransfer.Controller is not ready I0311 03:22:25.032823 1 tasks_processing.go:74] worker 22 stopped. I0311 03:22:25.033311 1 controller.go:203] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0311 03:22:25.033320 1 controller.go:203] Source periodic-conditional *controllerstatus.Simple is not ready I0311 03:22:25.033353 1 tasks_processing.go:74] worker 2 stopped. I0311 03:22:25.033685 1 controller.go:458] The operator is still being initialized I0311 03:22:25.033700 1 controller.go:481] The operator is healthy I0311 03:22:25.033981 1 recorder.go:70] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=abbad405f7cae56a0e4f232040719922f6123b300e93e99dc5f371b6a28924e3 I0311 03:22:25.034130 1 recorder.go:70] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=9207864de1b4f895501ed2a8d6c6c7270024458a5c8a4a9a618df4893b6b9d37 I0311 03:22:25.034184 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 35.989187ms to process 2 records I0311 03:22:25.034276 1 tasks_processing.go:74] worker 25 stopped. I0311 03:22:25.034415 1 recorder.go:70] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=5392c708b4cad310fa4d3733baec4e75f0854838e4524a6f9ce69d6e905c5cfa I0311 03:22:25.034465 1 recorder.go:70] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=b9ce2df8b431a1456cb829858ddd6a1ffd78db1ad3124f92300e7bfe96cd5960 I0311 03:22:25.034507 1 recorder.go:70] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=c5d7afdc8317dcb3229a7220161f60131d0011388e4752afb72bce1d2ddf3c9c I0311 03:22:25.034520 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 26.820685ms to process 3 records I0311 03:22:25.034611 1 tasks_processing.go:74] worker 0 stopped. I0311 03:22:25.034658 1 recorder.go:70] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=df0758d90faa9dfbc50a7560098040ad408b2d42582893e15855d724fcc06ea7 I0311 03:22:25.034717 1 recorder.go:70] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=9ffdf79b7365802741f27c17b7af810ff94cbd61ecab790e2591446bdba0835c I0311 03:22:25.034796 1 recorder.go:70] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=9a293a2955e6201c2920eb185d105a0e126330ba3b9c79659b222dabcde5782b I0311 03:22:25.034831 1 recorder.go:70] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=98c5cf0007ae7010344b1156c051414888c19a76360bfa0541c9fcd257df21a8 I0311 03:22:25.034865 1 recorder.go:70] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=808be40af1a9fcb70257102f4ceb86b675831c04241be7f7775fe5826da47458 I0311 03:22:25.034907 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=70fcb891fb767b20ef5656b51e9740f2543445a57f550820d0422057edfa7222 I0311 03:22:25.034948 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=6b6f84ce3eb041d06bf21181e561e5aa801491112efaa33b2dc29ca6d6c5cd36 I0311 03:22:25.035002 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=f4d8bf81af43a8a30ea3db5e6837e9c26e2578a270e8b0488eaa12d37704957c I0311 03:22:25.035054 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=64f52b4f963f53e4c8270615a6657a608471644a52a583dc501c2150f1ae7c4f I0311 03:22:25.035095 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=66e966c6635445ff99d9f1ce9d063924b912c8c8e95fe6b0ef619972a5863be7 I0311 03:22:25.035136 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=4a9efa1ada86e4cbb175f7ec0c97c44b4fb8ba2520ad99634358fa46188404f6 I0311 03:22:25.035176 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=945ea0b7ec40056419f01a05d8f917909fb6c536a68529d6f9effe575196485d I0311 03:22:25.035188 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 32.791946ms to process 12 records I0311 03:22:25.035277 1 tasks_processing.go:74] worker 11 stopped. I0311 03:22:25.035651 1 recorder.go:70] Recording config/version with fingerprint=fffa087ef5f4eae13e2e89cced67ece05aa31de219bb5df7b0506bddb6d33307 I0311 03:22:25.035666 1 recorder.go:70] Recording config/id with fingerprint=4ec1ab798765f1b0d9ce7258e59b1a1d30f3887cba4fd123c0725f95d0b885f6 I0311 03:22:25.035694 1 gather.go:177] gatherer "clusterconfig" function "version" took 39.57685ms to process 2 records I0311 03:22:25.035743 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 8.819495ms to process 0 records I0311 03:22:25.035812 1 tasks_processing.go:74] worker 15 stopped. I0311 03:22:25.035880 1 tasks_processing.go:74] worker 31 stopped. I0311 03:22:25.036009 1 recorder.go:70] Recording config/oauth with fingerprint=139d0f946408bee390c663409aebc1df4ccc685d726f530321e8ff7edf4e0a73 I0311 03:22:25.036023 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 20.610386ms to process 1 records I0311 03:22:25.036282 1 tasks_processing.go:74] worker 30 stopped. I0311 03:22:25.036298 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 8.506617ms to process 0 records I0311 03:22:25.036324 1 tasks_processing.go:74] worker 28 stopped. I0311 03:22:25.036342 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 11.402908ms to process 0 records I0311 03:22:25.036474 1 tasks_processing.go:74] worker 1 stopped. I0311 03:22:25.036534 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 9.777297ms to process 0 records E0311 03:22:25.036567 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0311 03:22:25.036636 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 10.752237ms to process 0 records I0311 03:22:25.036650 1 tasks_processing.go:74] worker 23 stopped. I0311 03:22:25.037048 1 tasks_processing.go:74] worker 21 stopped. I0311 03:22:25.037064 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 38.059317ms to process 0 records I0311 03:22:25.037514 1 tasks_processing.go:74] worker 8 stopped. I0311 03:22:25.037684 1 recorder.go:70] Recording config/ingress with fingerprint=1af056cc32fd64ea990558552f5db8064771c74271ac1ae2ea75e70cb9d82175 I0311 03:22:25.037702 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 11.488225ms to process 1 records I0311 03:22:25.039515 1 tasks_processing.go:74] worker 9 stopped. I0311 03:22:25.039633 1 recorder.go:70] Recording config/image with fingerprint=a639384d32fda4986ee0e019ad6d913f7f8bb1a559355d9f0cef5a7465d20f88 I0311 03:22:25.039652 1 gather.go:177] gatherer "clusterconfig" function "image" took 13.054209ms to process 1 records I0311 03:22:25.039753 1 tasks_processing.go:74] worker 18 stopped. I0311 03:22:25.039864 1 recorder.go:70] Recording config/featuregate with fingerprint=1b9f6efb81a6d788b9bc83d17623fbeb05bdbb04c834c6f0e56d32342dc1df65 I0311 03:22:25.039885 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 13.7667ms to process 1 records E0311 03:22:25.045626 1 cluster_transfer.go:95] failed to pull cluster transfer: unable to retrieve cluster transfer data from https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/: Get "https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/?search=cluster_uuid+is+%27fa66d3f1-4f5b-46f3-88a5-7aa8195b4be1%27+and+status+is+%27accepted%27": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.9:43140->172.30.0.10:53: read: connection refused I0311 03:22:25.045644 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=Disconnected message=failed to pull cluster transfer: unable to retrieve cluster transfer data from https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/: Get "https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/?search=cluster_uuid+is+%27fa66d3f1-4f5b-46f3-88a5-7aa8195b4be1%27+and+status+is+%27accepted%27": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.9:43140->172.30.0.10:53: read: connection refused I0311 03:22:25.047045 1 tasks_processing.go:74] worker 24 stopped. I0311 03:22:25.047958 1 recorder.go:70] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=fd289126983cf5c5b9a09d003c93888446658fb9d0bf58fac4640bf0d9afda5e I0311 03:22:25.048347 1 recorder.go:70] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=cd6d14e2393fea6b18bc63f95400713dfd3edbe471d5756dfb223676ccc1672a I0311 03:22:25.048368 1 gather.go:177] gatherer "clusterconfig" function "crds" took 21.277837ms to process 2 records I0311 03:22:25.048617 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0311 03:22:25.050430 1 tasks_processing.go:74] worker 17 stopped. I0311 03:22:25.050512 1 recorder.go:70] Recording config/olm_operators with fingerprint=80d3d107ec4009f3981891b83df9445c9400214bef5712a78c233e96fcb2fd1d I0311 03:22:25.050530 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 22.938173ms to process 1 records I0311 03:22:25.052678 1 tasks_processing.go:74] worker 14 stopped. I0311 03:22:25.052740 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 29.703406ms to process 0 records W0311 03:22:25.055979 1 sca.go:161] Failed to pull SCA certs from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: unable to retrieve SCA certs data from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: Post "https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.9:48466->172.30.0.10:53: read: connection refused I0311 03:22:25.055999 1 controllerstatus.go:80] name=scaController healthy=true reason=NonHTTPError message=Failed to pull SCA certs from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: unable to retrieve SCA certs data from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: Post "https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.9:48466->172.30.0.10:53: read: connection refused I0311 03:22:25.061732 1 tasks_processing.go:74] worker 29 stopped. E0311 03:22:25.061748 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0311 03:22:25.061754 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2ov8p207jqstd62uruk6mrk9u6msglar-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2ov8p207jqstd62uruk6mrk9u6msglar-primary-cert-bundle-secret" not found I0311 03:22:25.061957 1 recorder.go:70] Recording aggregated/ingress_controllers_certs with fingerprint=31576510fa7b992604c32a97cfe5b8f2bf304d41a034b8b37a121a5a10be414e I0311 03:22:25.062030 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 66.782174ms to process 1 records I0311 03:22:25.062124 1 tasks_processing.go:74] worker 19 stopped. E0311 03:22:25.062174 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0311 03:22:25.062256 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0311 03:22:25.062297 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0311 03:22:25.062330 1 recorder.go:70] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=295a2bc7810c628251f1f312c209b5eb4da888326706c635ac0c12aec0969e17 I0311 03:22:25.062400 1 recorder.go:70] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0311 03:22:25.062441 1 recorder.go:70] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0311 03:22:25.062471 1 recorder.go:70] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=fe7abfa7f7aea852d8bca6b7df9b4d5de32045254ff2774f9bd30f0b6dcb7dc4 I0311 03:22:25.062524 1 recorder.go:70] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0311 03:22:25.062635 1 recorder.go:70] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0311 03:22:25.062666 1 recorder.go:70] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0311 03:22:25.062692 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 57.491726ms to process 7 records I0311 03:22:25.063812 1 tasks_processing.go:74] worker 16 stopped. I0311 03:22:25.065704 1 recorder.go:70] Recording config/pod/openshift-console-operator/console-operator-85b84f584d-cjbtd with fingerprint=b74dada55f2de45b6b2fffaee17e0d47a056fb62aebea3330c43360217eaef50 I0311 03:22:25.065944 1 recorder.go:70] Recording config/pod/openshift-ovn-kubernetes/ovnkube-node-qtxj4 with fingerprint=90aa0335b0f9c975b23334dcfddb200447e6f8fb3b36c560199ae9aab34b5e47 I0311 03:22:25.066012 1 recorder.go:70] Recording config/running_containers with fingerprint=0aedb3b8ee1399b85221ccd97b008c6137ec273c9a761ae5ec74dbce39890c96 I0311 03:22:25.066021 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 58.473421ms to process 3 records I0311 03:22:25.066032 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 38.395288ms to process 0 records I0311 03:22:25.066038 1 tasks_processing.go:74] worker 12 stopped. I0311 03:22:25.066060 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0311 03:22:25.066135 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0311 03:22:25.066375 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0311 03:22:25.071365 1 base_controller.go:82] Caches are synced for ConfigController I0311 03:22:25.071376 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0311 03:22:25.075671 1 configmapobserver.go:84] configmaps "insights-config" not found I0311 03:22:25.105609 1 base_controller.go:82] Caches are synced for LoggingSyncer I0311 03:22:25.105622 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0311 03:22:25.851393 1 gather_cluster_operators.go:184] Unable to get operatorpkis.network.operator.openshift.io resource due to: operatorpkis.network.operator.openshift.io "ovn" is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot get resource "operatorpkis" in API group "network.operator.openshift.io" in the namespace "openshift-ovn-kubernetes" W0311 03:22:26.008135 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0311 03:22:26.051448 1 gather_cluster_operators.go:184] Unable to get operatorpkis.network.operator.openshift.io resource due to: operatorpkis.network.operator.openshift.io "signer" is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot get resource "operatorpkis" in API group "network.operator.openshift.io" in the namespace "openshift-ovn-kubernetes" I0311 03:22:26.235429 1 gather_cluster_operator_pods_and_events.go:121] Found 25 pods with 28 containers I0311 03:22:26.235448 1 gather_cluster_operator_pods_and_events.go:235] Maximum buffer size: 898779 bytes I0311 03:22:26.235664 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for console-operator container console-operator-85b84f584d-cjbtd pod in namespace openshift-console-operator (previous: true). I0311 03:22:26.439433 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for console-operator container console-operator-85b84f584d-cjbtd pod in namespace openshift-console-operator (previous: false). I0311 03:22:26.642126 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for console container console-5759fc9c4b-6h9w7 pod in namespace openshift-console (previous: false). I0311 03:22:26.839943 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for console container console-5759fc9c4b-z6bsd pod in namespace openshift-console (previous: false). I0311 03:22:26.852749 1 gather_cluster_operators.go:184] Unable to get configs.samples.operator.openshift.io resource due to: configs.samples.operator.openshift.io "cluster" not found W0311 03:22:27.008336 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0311 03:22:27.035440 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0311 03:22:27.035456 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for console container console-6f95d4c595-dmpbd pod in namespace openshift-console (previous: false). I0311 03:22:27.240789 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for console container console-df7dfbcc7-jcfr5 pod in namespace openshift-console (previous: false). I0311 03:22:27.439058 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for download-server container downloads-7b98f8c767-52dtt pod in namespace openshift-console (previous: false). I0311 03:22:27.454982 1 tasks_processing.go:74] worker 7 stopped. I0311 03:22:27.455094 1 recorder.go:70] Recording config/clusteroperator/console with fingerprint=ec6c85d1ee4b9dcc9d86f06a7fcf66cd5c8884ce58075e93021ea7f127cfdb12 I0311 03:22:27.455142 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0311 03:22:27.455168 1 recorder.go:70] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=d502239aa894e78ea5ef1b1c86dd1949f27639838766d2075a06d6121cedd042 I0311 03:22:27.455179 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0311 03:22:27.455200 1 recorder.go:70] Recording config/clusteroperator/dns with fingerprint=71e0856768db5df8bfe64390eb622681ec45b50bed6d1c9d46f8bebe857151bc I0311 03:22:27.455218 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0311 03:22:27.455258 1 recorder.go:70] Recording config/clusteroperator/image-registry with fingerprint=0de97347c8792f83b3203bd1320b857b2a684a120de4c99b15f21a8c29152abe I0311 03:22:27.455286 1 recorder.go:70] Recording config/clusteroperator/ingress with fingerprint=821747622330e1f39166eb1e28e0c7ba215caaf4ac2375be8c6e95c974f12430 I0311 03:22:27.455322 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=3e155738f71e8a3e9c8b9a5af9d782d7b2b0c8f35bd94f9012add19a01da712b I0311 03:22:27.455354 1 recorder.go:70] Recording config/clusteroperator/insights with fingerprint=5a589d22518889e8b62a53deab2b59684e450ddefaff7d9b12328e15b142091b I0311 03:22:27.455642 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=b1327411d54922aa975a0de7e6e92e196cece3505a95027099061c9c32508f94 I0311 03:22:27.455661 1 recorder.go:70] Recording config/clusteroperator/kube-apiserver with fingerprint=c680aedd16a69e3db91fbf72056f4f18b5c5309dd6f1d97ffc242a8a4023d4e8 I0311 03:22:27.455672 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0311 03:22:27.455688 1 recorder.go:70] Recording config/clusteroperator/kube-controller-manager with fingerprint=793c6aa6166b563535ce1650eda7ef4fa548dc2f5a903bd8730ccefe00414d41 I0311 03:22:27.455699 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0311 03:22:27.455715 1 recorder.go:70] Recording config/clusteroperator/kube-scheduler with fingerprint=54ccb71b9957b9b58df8af3a626c968626d8a7c8886269bc8a8d1604fba0d94e I0311 03:22:27.455726 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0311 03:22:27.455754 1 recorder.go:70] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=4039e9182242f2ea30993f81fc74ee52d32704db801785f880646293c1571012 I0311 03:22:27.455765 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0311 03:22:27.455784 1 recorder.go:70] Recording config/clusteroperator/monitoring with fingerprint=516df19c9d2e1b032b343b50ca6ed690aa65a1d1f7391db2f93c334fd379f3fb I0311 03:22:27.455886 1 recorder.go:70] Recording config/clusteroperator/network with fingerprint=1e840b90c09ee819eeb61e18bac329f995dfbca5cf0b9911bfae32ca0c60731c I0311 03:22:27.455917 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0311 03:22:27.455943 1 recorder.go:70] Recording config/clusteroperator/node-tuning with fingerprint=1ed42c99d04927dfc3d5f932c90a26c72833cd4894a1fae68fc8b4a2217b580e I0311 03:22:27.455974 1 recorder.go:70] Recording config/clusteroperator/openshift-apiserver with fingerprint=83c50d91d3b6acd8cb1df1d4ce1116a6898f9fcf63ebd571d519e0d85669fa85 I0311 03:22:27.455986 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0311 03:22:27.456003 1 recorder.go:70] Recording config/clusteroperator/openshift-controller-manager with fingerprint=4bb76690824182e4c50b9e091f675343df4061f14996fc83ed0931a4e5caf78a I0311 03:22:27.456014 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0311 03:22:27.456027 1 recorder.go:70] Recording config/clusteroperator/openshift-samples with fingerprint=906fee0b62f5cba9019ad18451469ca6d792d1ec2d7992b9dd38356ab556ffa0 I0311 03:22:27.456043 1 recorder.go:70] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=a36e55681af707ed275eed6ad3a23b8aeb68fff56f21fb59dad68d858eb5ef7d I0311 03:22:27.456059 1 recorder.go:70] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=769a5bff2f59ff32bd4d6c49d9330160d77daa98df270323bda600506f7241bd I0311 03:22:27.456075 1 recorder.go:70] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=0f379e1256cae6b34fdc6096494e284307f7281d84c5707cd5500f0b8cdc1cea I0311 03:22:27.456098 1 recorder.go:70] Recording config/clusteroperator/service-ca with fingerprint=5210326411d44892e6d109032e3bf0a15e7bc6af0dc08efba3888cd819f640c7 I0311 03:22:27.456108 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0311 03:22:27.456135 1 recorder.go:70] Recording config/clusteroperator/storage with fingerprint=f3bc1588e0f59c98686a5e98d8761262f8c9a19cfa8aa3dc40f82adf553dc1ab I0311 03:22:27.456153 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0311 03:22:27.456162 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0311 03:22:27.456169 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.4606768s to process 36 records I0311 03:22:27.643765 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for download-server container downloads-7b98f8c767-5r5hk pod in namespace openshift-console (previous: false). I0311 03:22:27.840132 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns container dns-default-fhk6g pod in namespace openshift-dns (previous: false). W0311 03:22:28.007726 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0311 03:22:28.039034 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-fhk6g pod in namespace openshift-dns for failing operator dns (previous: false): "container \"dns\" in pod \"dns-default-fhk6g\" is waiting to start: ContainerCreating" I0311 03:22:28.039049 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"dns\" in pod \"dns-default-fhk6g\" is waiting to start: ContainerCreating" I0311 03:22:28.039057 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-rbac-proxy container dns-default-fhk6g pod in namespace openshift-dns (previous: false). I0311 03:22:28.239018 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-fhk6g pod in namespace openshift-dns for failing operator kube-rbac-proxy (previous: false): "container \"kube-rbac-proxy\" in pod \"dns-default-fhk6g\" is waiting to start: ContainerCreating" I0311 03:22:28.239036 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"kube-rbac-proxy\" in pod \"dns-default-fhk6g\" is waiting to start: ContainerCreating" I0311 03:22:28.239076 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns container dns-default-jb8jn pod in namespace openshift-dns (previous: false). I0311 03:22:28.438966 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-jb8jn pod in namespace openshift-dns for failing operator dns (previous: false): "container \"dns\" in pod \"dns-default-jb8jn\" is waiting to start: ContainerCreating" I0311 03:22:28.438983 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"dns\" in pod \"dns-default-jb8jn\" is waiting to start: ContainerCreating" I0311 03:22:28.438991 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-rbac-proxy container dns-default-jb8jn pod in namespace openshift-dns (previous: false). I0311 03:22:28.639876 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-jb8jn pod in namespace openshift-dns for failing operator kube-rbac-proxy (previous: false): "container \"kube-rbac-proxy\" in pod \"dns-default-jb8jn\" is waiting to start: ContainerCreating" I0311 03:22:28.639894 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"kube-rbac-proxy\" in pod \"dns-default-jb8jn\" is waiting to start: ContainerCreating" I0311 03:22:28.639924 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns container dns-default-nl8qm pod in namespace openshift-dns (previous: false). I0311 03:22:28.839897 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-nl8qm pod in namespace openshift-dns for failing operator dns (previous: false): "container \"dns\" in pod \"dns-default-nl8qm\" is waiting to start: ContainerCreating" I0311 03:22:28.839917 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"dns\" in pod \"dns-default-nl8qm\" is waiting to start: ContainerCreating" I0311 03:22:28.839924 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-rbac-proxy container dns-default-nl8qm pod in namespace openshift-dns (previous: false). W0311 03:22:29.007966 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0311 03:22:29.039473 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-nl8qm pod in namespace openshift-dns for failing operator kube-rbac-proxy (previous: false): "container \"kube-rbac-proxy\" in pod \"dns-default-nl8qm\" is waiting to start: ContainerCreating" I0311 03:22:29.039491 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"kube-rbac-proxy\" in pod \"dns-default-nl8qm\" is waiting to start: ContainerCreating" I0311 03:22:29.039502 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-dxppk pod in namespace openshift-dns (previous: false). I0311 03:22:29.241290 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0311 03:22:29.241309 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-nsbsf pod in namespace openshift-dns (previous: false). I0311 03:22:29.439537 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0311 03:22:29.439554 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-vtzzs pod in namespace openshift-dns (previous: false). I0311 03:22:29.639135 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0311 03:22:29.639159 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-667cb477fd-6rzsx pod in namespace openshift-image-registry (previous: false). I0311 03:22:29.842778 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-667cb477fd-fmqzn pod in namespace openshift-image-registry (previous: false). W0311 03:22:30.003797 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0311 03:22:30.003828 1 tasks_processing.go:74] worker 5 stopped. E0311 03:22:30.003841 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0311 03:22:30.003852 1 recorder.go:70] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0311 03:22:30.003863 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.010251904s to process 1 records I0311 03:22:30.040705 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-6fbf5c974d-9wrzd pod in namespace openshift-image-registry (previous: false). I0311 03:22:30.238991 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for image-registry-6fbf5c974d-9wrzd pod in namespace openshift-image-registry for failing operator registry (previous: false): "container \"registry\" in pod \"image-registry-6fbf5c974d-9wrzd\" is waiting to start: ContainerCreating" I0311 03:22:30.239009 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"registry\" in pod \"image-registry-6fbf5c974d-9wrzd\" is waiting to start: ContainerCreating" I0311 03:22:30.239019 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-b7ks7 pod in namespace openshift-image-registry (previous: false). I0311 03:22:30.439643 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0311 03:22:30.439660 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-gv44s pod in namespace openshift-image-registry (previous: false). I0311 03:22:30.639481 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-jznzq pod in namespace openshift-image-registry (previous: false). I0311 03:22:30.840366 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-65b7f74665-4qrcb pod in namespace openshift-ingress (previous: false). I0311 03:22:31.046286 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-7cf68cfb5b-pscfz pod in namespace openshift-ingress (previous: false). I0311 03:22:31.241045 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-7cf68cfb5b-xczw6 pod in namespace openshift-ingress (previous: false). I0311 03:22:31.439706 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-6pth9 pod in namespace openshift-ingress-canary (previous: false). I0311 03:22:31.639037 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for ingress-canary-6pth9 pod in namespace openshift-ingress-canary for failing operator serve-healthcheck-canary (previous: false): "container \"serve-healthcheck-canary\" in pod \"ingress-canary-6pth9\" is waiting to start: ContainerCreating" I0311 03:22:31.639058 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"serve-healthcheck-canary\" in pod \"ingress-canary-6pth9\" is waiting to start: ContainerCreating" I0311 03:22:31.639089 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-dtkkr pod in namespace openshift-ingress-canary (previous: false). I0311 03:22:31.839434 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for ingress-canary-dtkkr pod in namespace openshift-ingress-canary for failing operator serve-healthcheck-canary (previous: false): "container \"serve-healthcheck-canary\" in pod \"ingress-canary-dtkkr\" is waiting to start: ContainerCreating" I0311 03:22:31.839454 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"serve-healthcheck-canary\" in pod \"ingress-canary-dtkkr\" is waiting to start: ContainerCreating" I0311 03:22:31.839493 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-kk4lh pod in namespace openshift-ingress-canary (previous: false). I0311 03:22:32.039479 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for ingress-canary-kk4lh pod in namespace openshift-ingress-canary for failing operator serve-healthcheck-canary (previous: false): "container \"serve-healthcheck-canary\" in pod \"ingress-canary-kk4lh\" is waiting to start: ContainerCreating" I0311 03:22:32.039496 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"serve-healthcheck-canary\" in pod \"ingress-canary-kk4lh\" is waiting to start: ContainerCreating" I0311 03:22:32.039512 1 tasks_processing.go:74] worker 4 stopped. I0311 03:22:32.039735 1 recorder.go:70] Recording events/openshift-console-operator with fingerprint=17aa8352a7fe6998e855d588e7e5d62f5c90ba8568bc87b10d7a2bfbea95d692 I0311 03:22:32.039858 1 recorder.go:70] Recording events/openshift-console with fingerprint=6e9f3ca6a83daedfc2f20de569717bf06bbc56206f8e9d9403657c5d99ff7f37 I0311 03:22:32.039881 1 recorder.go:70] Recording events/openshift-dns-operator with fingerprint=d84f2331d2f35255eba8377359676110b90ab7019d8c4faa1694504aa3843386 I0311 03:22:32.039925 1 recorder.go:70] Recording events/openshift-dns with fingerprint=66abcdde8d0fe5d69b758e923f28174bae2b6e6ad1d5e041722d663a3c79069e I0311 03:22:32.040040 1 recorder.go:70] Recording events/openshift-image-registry with fingerprint=af0ee36814613dad7b38c666ae2f854ec05d76b8c23b4dc5ddeba617593bc05b I0311 03:22:32.040068 1 recorder.go:70] Recording events/openshift-ingress-operator with fingerprint=b9326b496207e8426a9c4fdb5a656f2622c1cd27c3e25b9547c56c7c3701d761 I0311 03:22:32.040146 1 recorder.go:70] Recording events/openshift-ingress with fingerprint=7937a9aa92fb1b1fc8e73959f276d0ed484afa8ff35b01e232b0d7e76f8798ed I0311 03:22:32.040165 1 recorder.go:70] Recording events/openshift-ingress-canary with fingerprint=580daae6c86486cb5f067e672b758dd2152b049eda0076c8c49a4ab6617e75ac I0311 03:22:32.040330 1 recorder.go:70] Recording config/pod/openshift-console-operator/console-operator-85b84f584d-cjbtd with fingerprint=b74dada55f2de45b6b2fffaee17e0d47a056fb62aebea3330c43360217eaef50 E0311 03:22:32.040352 1 gather.go:161] error recording gatherer "clusterconfig" function "operators_pods_and_events" result "config/pod/openshift-console-operator/console-operator-85b84f584d-cjbtd.json" because of the error: the record with the same name "config/pod/openshift-console-operator/console-operator-85b84f584d-cjbtd.json" was already recorded and had the fingerprint "b74dada55f2de45b6b2fffaee17e0d47a056fb62aebea3330c43360217eaef50", overwriting with the record having fingerprint "b74dada55f2de45b6b2fffaee17e0d47a056fb62aebea3330c43360217eaef50" W0311 03:22:32.040365 1 gather.go:155] issue recording gatherer "clusterconfig" function "operators_pods_and_events" result "config/pod/openshift-console-operator/console-operator-85b84f584d-cjbtd.json" because of the warning: warning: the record with the same fingerprint "b74dada55f2de45b6b2fffaee17e0d47a056fb62aebea3330c43360217eaef50" was already recorded at path "config/pod/openshift-console-operator/console-operator-85b84f584d-cjbtd.json", recording another one with a different path "config/pod/openshift-console-operator/console-operator-85b84f584d-cjbtd.json" I0311 03:22:32.040381 1 recorder.go:70] Recording config/pod/openshift-console-operator/logs/console-operator-85b84f584d-cjbtd/console-operator_previous.log with fingerprint=c6cacc35e12148e10ca3f4de5f43192c9c41e869c251eb39ccfa43910af96b1f I0311 03:22:32.040550 1 recorder.go:70] Recording config/pod/openshift-console-operator/logs/console-operator-85b84f584d-cjbtd/console-operator_current.log with fingerprint=34b6be21a772df0c07551760fc0c200bb0ff10493206fdb75eec8f3d3a76ac1e I0311 03:22:32.040564 1 recorder.go:70] Recording config/pod/openshift-console/logs/console-5759fc9c4b-6h9w7/console_current.log with fingerprint=0bec5818de4a5b4888c9b0bc31d37aeaa4081fdae88c58c96929eb4c3008fec8 I0311 03:22:32.040574 1 recorder.go:70] Recording config/pod/openshift-console/logs/console-6f95d4c595-dmpbd/console_current.log with fingerprint=42627013191267b33d3a6e182c71bc53250cc99adfff3815429636f5a52d1b22 I0311 03:22:32.040586 1 recorder.go:70] Recording config/pod/openshift-console/logs/console-df7dfbcc7-jcfr5/console_current.log with fingerprint=9c83c7953c3c80fddc0d33231aa25f490f8fd6837a0b8f249c01e564de1b27c4 I0311 03:22:32.040592 1 recorder.go:70] Recording config/pod/openshift-console/logs/downloads-7b98f8c767-52dtt/download-server_current.log with fingerprint=f15d16d2ab948984761f09df0b84eb92afb60848d7d3346f1eb09f4312a3b552 I0311 03:22:32.040598 1 recorder.go:70] Recording config/pod/openshift-console/logs/downloads-7b98f8c767-5r5hk/download-server_current.log with fingerprint=a99a437c5a1ffea8da5f9c773299f2940c82a2f1e9b8c5392fdd5e3773fcd84a I0311 03:22:32.040694 1 recorder.go:70] Recording config/pod/openshift-dns/dns-default-fhk6g with fingerprint=dcd3b9e27442098cdfce1b42b24155b5517716105470f1bf8c14f1f52a2d3a32 I0311 03:22:32.040765 1 recorder.go:70] Recording config/pod/openshift-dns/dns-default-jb8jn with fingerprint=19efdfb7973a13867c0a247bea3b60ba2c0fda8eaa13aeecc85b287b77e80b2d I0311 03:22:32.040834 1 recorder.go:70] Recording config/pod/openshift-dns/dns-default-nl8qm with fingerprint=efd5ad750a1976b5af4ee660b28a83748303b200a2b777b3fb4b667c2da3114f I0311 03:22:32.040865 1 recorder.go:70] Recording config/pod/openshift-image-registry/logs/image-registry-667cb477fd-6rzsx/registry_current.log with fingerprint=d5ddda2d06921789c8ffc5167101691d9c1894b736a1b4711cd63201739a76e6 I0311 03:22:32.040896 1 recorder.go:70] Recording config/pod/openshift-image-registry/logs/image-registry-667cb477fd-fmqzn/registry_current.log with fingerprint=cba5b9bc0f05f77e64d4c694c87d861ab51325090c61ed434c337489ec4f5da8 I0311 03:22:32.040994 1 recorder.go:70] Recording config/pod/openshift-image-registry/image-registry-6fbf5c974d-9wrzd with fingerprint=faea500a100ee81437613505f6be325c58cde46792daaae6ae343cbc28c66b85 I0311 03:22:32.041003 1 recorder.go:70] Recording config/pod/openshift-image-registry/logs/node-ca-gv44s/node-ca_current.log with fingerprint=84ebd74af58efcca190d780efe7feca96d447500f9244ca25a17fef359a44bd1 I0311 03:22:32.041010 1 recorder.go:70] Recording config/pod/openshift-image-registry/logs/node-ca-jznzq/node-ca_current.log with fingerprint=ce12869f33665fd3c180d05789f41bb0d848d4a6639409711cc8827d1c203c87 I0311 03:22:32.041039 1 recorder.go:70] Recording config/pod/openshift-ingress/logs/router-default-65b7f74665-4qrcb/router_current.log with fingerprint=a795d32520254a2afc44f07d4ab927b755cde570a41c6f04f52ac90cb2dda76a I0311 03:22:32.041073 1 recorder.go:70] Recording config/pod/openshift-ingress/logs/router-default-7cf68cfb5b-pscfz/router_current.log with fingerprint=4f31acf4377b9c57afca5b3590f2d58ddd5f6ead1e94e9b727f67b260976b8d6 I0311 03:22:32.041109 1 recorder.go:70] Recording config/pod/openshift-ingress/logs/router-default-7cf68cfb5b-xczw6/router_current.log with fingerprint=55bc54c8d24206f175d2a8547573f89e51b4eb173bb20b90566def6d39f6e5ca I0311 03:22:32.041166 1 recorder.go:70] Recording config/pod/openshift-ingress-canary/ingress-canary-6pth9 with fingerprint=042ec948cbc7306abdf2ffd059efad2dec3e35180dfd261eb5cc4a2c30180055 I0311 03:22:32.041220 1 recorder.go:70] Recording config/pod/openshift-ingress-canary/ingress-canary-dtkkr with fingerprint=d28eca62b489273c2716320b065c8ed308b9d59b369c66a3ad187022da7bdaf9 I0311 03:22:32.041287 1 recorder.go:70] Recording config/pod/openshift-ingress-canary/ingress-canary-kk4lh with fingerprint=5345008af8b11ea74bffcd168b0ebb0901d5885b04b6dda12404601401ccff01 I0311 03:22:32.041296 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 7.0460151s to process 30 records I0311 03:22:37.838710 1 tasks_processing.go:74] worker 26 stopped. I0311 03:22:37.838751 1 recorder.go:70] Recording config/installplans with fingerprint=f17dbfacc3bfddf27ca3b213b39495434cd4c4e9e3dbd69566ffb3845bbcf539 I0311 03:22:37.838763 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.843356037s to process 1 records I0311 03:22:38.608911 1 tasks_processing.go:74] worker 10 stopped. I0311 03:22:38.609275 1 recorder.go:70] Recording config/serviceaccounts with fingerprint=73582af525bf551f979ef96bfb3cbb0257826668d63659ccecb8b26ded60e307 I0311 03:22:38.609296 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.605556889s to process 1 records E0311 03:22:38.609384 1 periodic.go:250] "Unhandled Error" err="clusterconfig failed after 13.616s with: function \"pod_network_connectivity_checks\" failed with an error, function \"tsdb_status\" failed with an error, function \"metrics\" failed with an error, function \"active_alerts\" failed with an error, function \"silenced_alerts\" failed with an error, function \"machines\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"machine_configs\" failed with an error, function \"support_secret\" failed with an error, function \"ingress_certificates\" failed with an error, function \"config_maps\" failed with an error, function \"dvo_metrics\" failed with an error, unable to record function \"operators_pods_and_events\" record \"config/pod/openshift-console-operator/console-operator-85b84f584d-cjbtd.json\"" I0311 03:22:38.610503 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "pod_network_connectivity_checks" failed with an error, function "tsdb_status" failed with an error, function "metrics" failed with an error, function "active_alerts" failed with an error, function "silenced_alerts" failed with an error, function "machines" failed with an error, function "machine_healthchecks" failed with an error, function "machine_configs" failed with an error, function "support_secret" failed with an error, function "ingress_certificates" failed with an error, function "config_maps" failed with an error, function "dvo_metrics" failed with an error, unable to record function "operators_pods_and_events" record "config/pod/openshift-console-operator/console-operator-85b84f584d-cjbtd.json" I0311 03:22:38.610522 1 periodic.go:212] Running workloads gatherer I0311 03:22:38.610542 1 tasks_processing.go:45] number of workers: 2 I0311 03:22:38.610555 1 tasks_processing.go:69] worker 1 listening for tasks. I0311 03:22:38.610560 1 tasks_processing.go:71] worker 1 working on workload_info task. I0311 03:22:38.610570 1 tasks_processing.go:69] worker 0 listening for tasks. I0311 03:22:38.610622 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0311 03:22:38.616364 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-165.ec2.internal... I0311 03:22:38.616372 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-47.ec2.internal... I0311 03:22:38.616407 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-207.ec2.internal... I0311 03:22:38.635689 1 tasks_processing.go:74] worker 0 stopped. I0311 03:22:38.635704 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 25.050532ms to process 0 records I0311 03:22:42.717008 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 4s I0311 03:22:42.740705 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0311 03:22:42.747443 1 gather_workloads_info.go:387] No image sha256:bae7a33f8db8a3d4b3c4c05498aba85a0ce463f85322067c48e79663710e616e (8ms) I0311 03:22:42.755043 1 gather_workloads_info.go:387] No image sha256:a7c71f3c9714cf1717c45e12ec817be6d6209f00989e6a31d634e527f0c8147d (8ms) I0311 03:22:42.762071 1 gather_workloads_info.go:387] No image sha256:4f175ee49f51dc4379e9993fecd1657c7a9c4c3dc096772cb198d0212b9eea47 (7ms) I0311 03:22:42.769160 1 gather_workloads_info.go:387] No image sha256:38dd36da63b1f01316a02daf6c18b821318c6ec313712147292d74abf955e0da (7ms) I0311 03:22:42.776135 1 gather_workloads_info.go:387] No image sha256:3d0634fe58641d5242649c44ddba70ed67fdd6d2dcb5c2261df5cee8b33de9fd (7ms) I0311 03:22:42.782994 1 gather_workloads_info.go:387] No image sha256:8ef4bdf07dda423fab73484dbc66d527ce8ba8cc2d4f99210bc1bc24ea08c0cb (7ms) I0311 03:22:42.790003 1 gather_workloads_info.go:387] No image sha256:5cba745f09f3bd6b8028c6773597cbb30bf3639a84f2b4b6493fe20642339a2c (7ms) I0311 03:22:42.796862 1 gather_workloads_info.go:387] No image sha256:23c4fe84047a4e6cbe7f75f470e7d6fe0e61e7910dd17945b1b61bc4b72f3f2a (7ms) I0311 03:22:42.803503 1 gather_workloads_info.go:387] No image sha256:944d9261ba7a143131fe8267c172defc5f37acc1cea3d4d373ec6fc5d8bfcc31 (7ms) I0311 03:22:42.810138 1 gather_workloads_info.go:387] No image sha256:0a92dd43975f972e3dc707fb37854e046813531553038147e9c90d54a8d9df73 (7ms) I0311 03:22:42.846582 1 gather_workloads_info.go:387] No image sha256:6d6ebff54b8adac74f4d1b12ac8aa0f16cd7b28370e0d6aa847d1c457e03a5b6 (36ms) I0311 03:22:42.947678 1 gather_workloads_info.go:387] No image sha256:2d077e73dd76873ee2a1583aa3b3e76a1b408737f0af2e22b2aa055604d89e81 (101ms) I0311 03:22:43.047707 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (100ms) I0311 03:22:43.148374 1 gather_workloads_info.go:387] No image sha256:7854174b4b98e9b10d9c8e9b43113df1217ab21db4b2395d9399af0c3d7911a3 (101ms) I0311 03:22:43.248169 1 gather_workloads_info.go:387] No image sha256:ceab8a8c340801fcebf800620d3fef5493d8a443077a21c6e56984051bd3abde (100ms) I0311 03:22:43.347975 1 gather_workloads_info.go:387] No image sha256:ccdda82b4adfa8ee4a84b3bad19693be6a42a9881a7b83788d835912129c49a4 (100ms) I0311 03:22:43.447610 1 gather_workloads_info.go:387] No image sha256:00f977c2275f5bf03d423911159a39ca44fabc1862ecb7ae492198ec8c46c6bb (100ms) I0311 03:22:43.547764 1 gather_workloads_info.go:387] No image sha256:e40bbab1e52f580ab471bd7ab19ae23a4bcbaf1893243a05238af6bede7ba9cc (100ms) I0311 03:22:43.648398 1 gather_workloads_info.go:387] No image sha256:929a4a9c96a08b57fe4c315deca55c30d1990c83e34b8b5692f7646d29f715c8 (101ms) I0311 03:22:43.747733 1 gather_workloads_info.go:387] No image sha256:48404bd61d05dc738ccce4d22e36b30dbcdc6015b06b4afb604fd1baeee35bf2 (99ms) I0311 03:22:43.848224 1 gather_workloads_info.go:387] No image sha256:68d8e92046969bcfcbca2d2b3dcf4dcad5f6edc5fcb44861eb22f8079a43ea4d (100ms) I0311 03:22:43.947807 1 gather_workloads_info.go:387] No image sha256:46c2c396ca651699738c3f26b79b10af2922123dd27c2e32382d2a31a87e92c7 (100ms) I0311 03:22:44.047548 1 gather_workloads_info.go:387] No image sha256:8b1e32ce43eb1247b0558230b6e0baa85ac62f1a5e2a2089d40b4fea90538529 (100ms) I0311 03:22:44.147790 1 gather_workloads_info.go:387] No image sha256:87efe06afd1f04426fca7f86c0f74c4ee75c311ba199ddabd5b849b877bc59fa (100ms) I0311 03:22:44.248937 1 gather_workloads_info.go:387] No image sha256:a3d34d8f38a540ce79d34a5248b1965346155497db57b72cb4758d659418059c (101ms) I0311 03:22:44.348154 1 gather_workloads_info.go:387] No image sha256:1cc2f3c83068c8c67d9070f67677e1bb6b4feff7ac5744cad08711893e56b55f (99ms) I0311 03:22:44.448040 1 gather_workloads_info.go:387] No image sha256:d28734effaadd66434e77a1a5fbe2e8a4ca2066cd9f8868c22ade9475539bfd7 (100ms) I0311 03:22:44.547571 1 gather_workloads_info.go:387] No image sha256:76f820bd9bd138d29305d545d7d49bfe63e2923f1fb1ec2e8eac81a388359024 (100ms) I0311 03:22:44.650579 1 gather_workloads_info.go:387] No image sha256:1f2aa1e1026570162984f7fbb634635bcf319e408cafa8f9034ee842a2f193ab (103ms) I0311 03:22:44.747379 1 gather_workloads_info.go:387] No image sha256:7941b5d9a758b8667e99cd9236f7f96ec036af61df5cbc96e16077d36700d7c7 (97ms) I0311 03:22:44.847169 1 gather_workloads_info.go:387] No image sha256:9697bc2258bdfa9ae8c1866cc7eb0b3b46851998db827b121bdf77417a881eb3 (100ms) I0311 03:22:44.947044 1 gather_workloads_info.go:387] No image sha256:0a8a0473029ace3adbb66f490dbe560f0e7782a38cdf32c1f7dd3e092e1d191e (100ms) I0311 03:22:45.047507 1 gather_workloads_info.go:387] No image sha256:373614619b9420b110d1508c5f17e066ce69c4c226fd04d02fbb959d9ba41eb6 (100ms) I0311 03:22:45.147471 1 gather_workloads_info.go:387] No image sha256:039479f355d6903e1018abf8427fece3465c4348d4de2144ec38114b51bec27e (100ms) I0311 03:22:45.247532 1 gather_workloads_info.go:387] No image sha256:420c12f61c53e54eab2d99476759c0de339fca98ce0a3a782bc6545cc0e97a9c (100ms) I0311 03:22:45.347948 1 gather_workloads_info.go:387] No image sha256:752be69b2262be713df12c47f4bac8c2dafed272c401e9a89f8060f053d68054 (100ms) I0311 03:22:45.447861 1 gather_workloads_info.go:387] No image sha256:270a7bac889ed64d75ce11ae26e8634a6209ff718b83fae2c888cfc758316aa3 (100ms) I0311 03:22:45.546840 1 gather_workloads_info.go:387] No image sha256:2fac754deaeade3456361eed52e344318ff16d04819384432759f0ea35530114 (99ms) I0311 03:22:45.647678 1 gather_workloads_info.go:387] No image sha256:bcca5c17371caa2dce7e28da4c9fe30163c340fa172187f94db90ea1bd6ab80d (101ms) I0311 03:22:45.747484 1 gather_workloads_info.go:387] No image sha256:6c3c7f77d7a3815bbb69cad5f6f1742c17d9597bc135ba3d116c9736d98fae33 (100ms) I0311 03:22:45.847684 1 gather_workloads_info.go:387] No image sha256:1d471ad866b8564b209214d7dd3a7363c47e9535211d660b63b46abb278af7ab (100ms) I0311 03:22:45.947557 1 gather_workloads_info.go:387] No image sha256:14703f73bb1ca69ea03b726e340b3b68f8e294e1f22c80d1a16666ea3d4a88a3 (100ms) I0311 03:22:46.047110 1 gather_workloads_info.go:387] No image sha256:f2c46054a8f64a0e949cbf30295f31b4a35a0203c6ad03fa7ec922b4101dcbf8 (100ms) I0311 03:22:46.147101 1 gather_workloads_info.go:387] No image sha256:59db91eadf1e1e5343777be43ff25fb712f300f9341be48ba05f45270944536f (100ms) I0311 03:22:46.247171 1 gather_workloads_info.go:387] No image sha256:18956d5fbc8d0369a26cb1e00eb5c3aff1973ae8cbd3983920a0cf2851f05358 (100ms) I0311 03:22:46.348513 1 gather_workloads_info.go:387] No image sha256:824a3b8f78e19aa21a9f6444aefe8e0b624886cca18fe828b13acbab55e6e868 (101ms) I0311 03:22:46.447459 1 gather_workloads_info.go:387] No image sha256:527ed405bf66234718321996dacf51f91c64c746de9f1509416ef6ce89c90a50 (99ms) I0311 03:22:46.548465 1 gather_workloads_info.go:387] No image sha256:6277dac65decba0ea094146bb99600f479eaf804696842e724de1f1b436f9a38 (101ms) I0311 03:22:46.548500 1 tasks_processing.go:74] worker 1 stopped. I0311 03:22:46.548970 1 recorder.go:70] Recording config/workload_info with fingerprint=820936724ab6e021185a88feb1c8f845c87e448c22105d352916e4628115eee0 I0311 03:22:46.548998 1 gather.go:177] gatherer "workloads" function "workload_info" took 7.937932665s to process 1 records I0311 03:22:46.549019 1 periodic.go:259] Periodic gather workloads completed in 7.938s I0311 03:22:46.549030 1 controllerstatus.go:80] name=periodic-workloads healthy=true reason= message= I0311 03:22:46.549037 1 periodic.go:212] Running conditional gatherer I0311 03:22:46.556546 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.19.9/gathering_rules I0311 03:22:46.563218 1 conditional_gatherer.go:107] Get "https://console.redhat.com/api/gathering/v2/4.19.9/gathering_rules": dial tcp: lookup console.redhat.com on 172.30.0.10:53: read udp 10.128.0.9:39252->172.30.0.10:53: read: connection refused I0311 03:22:46.563554 1 conditional_gatherer.go:336] updating alerts cache for conditional gatherer E0311 03:22:46.566728 1 conditional_gatherer.go:322] unable to update alerts cache: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?match%5B%5D=ALERTS%7Balertstate%3D%22firing%22%7D&query=ALERTS": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: read udp 10.128.0.9:49623->172.30.0.10:53: read: connection refused I0311 03:22:46.566792 1 conditional_gatherer.go:382] updating version cache for conditional gatherer I0311 03:22:46.574032 1 conditional_gatherer.go:390] cluster version is '4.19.9' E0311 03:22:46.574047 1 conditional_gatherer.go:207] error checking conditions for a gathering rule: alerts cache is missing E0311 03:22:46.574053 1 conditional_gatherer.go:207] error checking conditions for a gathering rule: alerts cache is missing E0311 03:22:46.574056 1 conditional_gatherer.go:207] error checking conditions for a gathering rule: alerts cache is missing E0311 03:22:46.574059 1 conditional_gatherer.go:207] error checking conditions for a gathering rule: alerts cache is missing E0311 03:22:46.574062 1 conditional_gatherer.go:207] error checking conditions for a gathering rule: alerts cache is missing E0311 03:22:46.574066 1 conditional_gatherer.go:207] error checking conditions for a gathering rule: alerts cache is missing E0311 03:22:46.574069 1 conditional_gatherer.go:207] error checking conditions for a gathering rule: alerts cache is missing E0311 03:22:46.574072 1 conditional_gatherer.go:207] error checking conditions for a gathering rule: alerts cache is missing E0311 03:22:46.574075 1 conditional_gatherer.go:207] error checking conditions for a gathering rule: alerts cache is missing I0311 03:22:46.574089 1 tasks_processing.go:45] number of workers: 3 I0311 03:22:46.574100 1 tasks_processing.go:69] worker 2 listening for tasks. I0311 03:22:46.574104 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0311 03:22:46.574109 1 tasks_processing.go:69] worker 0 listening for tasks. I0311 03:22:46.574123 1 tasks_processing.go:69] worker 1 listening for tasks. I0311 03:22:46.574142 1 tasks_processing.go:71] worker 1 working on conditional_gatherer_rules task. I0311 03:22:46.574161 1 tasks_processing.go:74] worker 1 stopped. I0311 03:22:46.574186 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0311 03:22:46.574321 1 recorder.go:70] Recording insights-operator/conditional-gatherer-rules with fingerprint=7034af97e7e41c22e4b775abdd4b9066c8ebb19da33eb7f69f39bfd2eb5f6406 I0311 03:22:46.574341 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.974µs to process 1 records I0311 03:22:46.574392 1 recorder.go:70] Recording insights-operator/remote-configuration with fingerprint=0394430c431eec4d48bb1811a90918e95161d2282c59af26f2473613cc0959db I0311 03:22:46.574406 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.105µs to process 1 records I0311 03:22:46.574415 1 tasks_processing.go:74] worker 0 stopped. I0311 03:22:46.574511 1 tasks_processing.go:74] worker 2 stopped. I0311 03:22:46.574523 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 401.269µs to process 0 records I0311 03:22:46.574541 1 controllerstatus.go:89] name=periodic-conditional healthy=false reason=NotAvailable message=Get "https://console.redhat.com/api/gathering/v2/4.19.9/gathering_rules": dial tcp: lookup console.redhat.com on 172.30.0.10:53: read udp 10.128.0.9:39252->172.30.0.10:53: read: connection refused I0311 03:22:46.574557 1 recorder.go:70] Recording insights-operator/remote-configuration.json with fingerprint=359de9c990c741675cec72fda96b5c3682221efdb4799f5eaa6e9805bcd3b5c1 I0311 03:22:46.597821 1 recorder.go:70] Recording insights-operator/gathers with fingerprint=e740b7fde56cefcb416db0153938912e9f8552eeecb695a62c761798b0082331 I0311 03:22:46.597945 1 diskrecorder.go:70] Writing 134 records to /var/lib/insights-operator/insights-2026-03-11-032246.tar.gz I0311 03:22:46.607953 1 diskrecorder.go:51] Wrote 134 records to disk in 10ms I0311 03:22:46.607984 1 periodic.go:281] Gathering cluster info every 2h0m0s I0311 03:22:46.607999 1 periodic.go:282] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0311 03:23:54.993848 1 diskrecorder.go:223] Found files to send: insights-2026-03-11-032246.tar.gz I0311 03:23:54.993878 1 insightsuploader.go:150] Checking archives to upload periodically every 16m26.293731213s I0311 03:23:54.993888 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0311 03:23:55.008372 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0311 03:23:55.145523 1 requests.go:87] Successfully reported id=2026-03-11T03:23:54Z x-rh-insights-request-id=4b4f3a1ebf2a4f6e936501b6492ca945, wrote=77409 I0311 03:23:55.145580 1 insightsuploader.go:187] Uploaded report successfully in 151.682621ms I0311 03:23:55.145605 1 controller.go:119] Initializing last reported time to 2026-03-11T03:23:54Z I0311 03:23:55.145615 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0311 03:23:55.145621 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0311 03:23:55.145627 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0311 03:23:55.150373 1 controller.go:481] The operator is healthy I0311 03:24:25.010387 1 controller.go:481] The operator is healthy I0311 03:24:55.631903 1 insightsreport.go:137] Pulling report from smart-proxy I0311 03:24:55.631932 1 insightsreport.go:149] Retrieving report I0311 03:24:55.639804 1 requests.go:110] Retrieving report for cluster: fa66d3f1-4f5b-46f3-88a5-7aa8195b4be1 I0311 03:24:55.639815 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/fa66d3f1-4f5b-46f3-88a5-7aa8195b4be1/reports I0311 03:24:55.642677 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/fa66d3f1-4f5b-46f3-88a5-7aa8195b4be1/reports I0311 03:24:55.796780 1 insightsreport.go:184] Report retrieved I0311 03:24:55.822297 1 insightsreport.go:239] Report retrieved correctly I0311 03:26:25.010463 1 controller.go:481] The operator is healthy I0311 03:27:24.986366 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0311 03:27:24.991165 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0311 03:27:24.991191 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0311 03:27:24.994643 1 secretconfigobserver.go:119] support secret does not exist I0311 03:28:25.010751 1 controller.go:481] The operator is healthy I0311 03:30:25.010941 1 controller.go:481] The operator is healthy I0311 03:32:24.987177 1 configmapobserver.go:84] configmaps "insights-config" not found I0311 03:32:24.995266 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0311 03:32:24.999165 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0311 03:32:24.999182 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0311 03:32:25.002870 1 secretconfigobserver.go:119] support secret does not exist I0311 03:32:25.014487 1 controller.go:481] The operator is healthy I0311 03:34:25.010881 1 controller.go:481] The operator is healthy I0311 03:36:25.011144 1 controller.go:481] The operator is healthy I0311 03:37:25.003652 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0311 03:37:25.008732 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0311 03:37:25.008750 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0311 03:37:25.012442 1 secretconfigobserver.go:119] support secret does not exist I0311 03:38:25.010964 1 controller.go:481] The operator is healthy I0311 03:40:25.010363 1 controller.go:481] The operator is healthy I0311 03:40:26.440200 1 insightsuploader.go:146] Nothing to report since 2026-03-11T03:23:54Z I0311 03:42:24.986358 1 configmapobserver.go:84] configmaps "insights-config" not found I0311 03:42:25.008993 1 controller.go:481] The operator is healthy I0311 03:42:25.012843 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0311 03:42:25.016627 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0311 03:42:25.016645 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0311 03:42:25.019916 1 secretconfigobserver.go:119] support secret does not exist I0311 03:44:25.009625 1 controller.go:481] The operator is healthy I0311 03:46:25.010633 1 controller.go:481] The operator is healthy I0311 03:47:25.020169 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0311 03:47:25.024978 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0311 03:47:25.025000 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0311 03:47:25.028359 1 secretconfigobserver.go:119] support secret does not exist I0311 03:48:25.015960 1 controller.go:481] The operator is healthy I0311 03:50:25.010818 1 controller.go:481] The operator is healthy I0311 03:52:24.986780 1 configmapobserver.go:84] configmaps "insights-config" not found I0311 03:52:24.989971 1 configmapobserver.go:84] configmaps "insights-config" not found I0311 03:52:25.009826 1 controller.go:481] The operator is healthy I0311 03:52:25.028632 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0311 03:52:25.032299 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0311 03:52:25.032316 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0311 03:52:25.035523 1 secretconfigobserver.go:119] support secret does not exist I0311 03:54:25.014020 1 controller.go:481] The operator is healthy I0311 03:56:11.539593 1 insightsuploader.go:146] Nothing to report since 2026-03-11T03:23:54Z I0311 03:56:25.010446 1 controller.go:481] The operator is healthy I0311 03:57:25.035714 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0311 03:57:25.042177 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0311 03:57:25.042197 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0311 03:57:25.046112 1 secretconfigobserver.go:119] support secret does not exist I0311 03:58:25.012627 1 controller.go:481] The operator is healthy I0311 04:00:25.010646 1 controller.go:481] The operator is healthy I0311 04:02:24.987444 1 configmapobserver.go:84] configmaps "insights-config" not found I0311 04:02:24.991228 1 configmapobserver.go:84] configmaps "insights-config" not found I0311 04:02:25.010100 1 controller.go:481] The operator is healthy I0311 04:02:25.047109 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0311 04:02:25.052080 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0311 04:02:25.052097 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0311 04:02:25.056294 1 secretconfigobserver.go:119] support secret does not exist I0311 04:04:25.010632 1 controller.go:481] The operator is healthy