W0331 03:52:07.366688       1 cmd.go:257] Using insecure, self-signed certificates
I0331 03:52:08.046501       1 start.go:138] Unable to read service ca bundle: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory
I0331 03:52:08.046796       1 observer_polling.go:159] Starting file observer
I0331 03:52:08.504861       1 operator.go:60] Starting insights-operator v0.0.0-master+$Format:%H$
I0331 03:52:08.505152       1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"}
I0331 03:52:08.505942       1 secure_serving.go:57] Forcing use of http/1.1 only
W0331 03:52:08.505965       1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected.
W0331 03:52:08.505970       1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected.
W0331 03:52:08.505974       1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected.
W0331 03:52:08.505978       1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected.
W0331 03:52:08.505980       1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected.
W0331 03:52:08.505983       1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected.
I0331 03:52:08.505988       1 simple_featuregate_reader.go:171] Starting feature-gate-detector
I0331 03:52:08.509532       1 operator.go:125] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GatewayAPI GatewayAPIController HighlyAvailableArbiter ImageVolume IngressControllerLBSubnetsAWS KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController SigstoreImageVerification StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPClusterHostedDNSInstall GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall ImageModeStatusReporting ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesAzure ManagedBootImagesvSphere MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutatingAdmissionPolicy NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterOperations NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PreconfiguredUDNAddresses SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeAttributesClass VolumeGroupSnapshot]
I0331 03:52:08.509591       1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"437efa7e-3963-4fbf-8753-bc2ac1458283", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}}
I0331 03:52:08.511853       1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController
I0331 03:52:08.511869       1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController
I0331 03:52:08.511901       1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
I0331 03:52:08.511909       1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
I0331 03:52:08.511917       1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
I0331 03:52:08.511922       1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
I0331 03:52:08.512197       1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/tmp/serving-cert-18288637/tls.crt::/tmp/serving-cert-18288637/tls.key"
I0331 03:52:08.512523       1 secure_serving.go:213] Serving securely on [::]:8443
I0331 03:52:08.512560       1 tlsconfig.go:243] "Starting DynamicServingCertificateController"
W0331 03:52:08.515569       1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used.
I0331 03:52:08.515855       1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status
I0331 03:52:08.515973       1 base_controller.go:76] Waiting for caches to sync for ConfigController
I0331 03:52:08.522245       1 secretconfigobserver.go:249] Found cloud.openshift.com token
I0331 03:52:08.522263       1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status
I0331 03:52:08.526607       1 secretconfigobserver.go:119] support secret does not exist
I0331 03:52:08.531436       1 secretconfigobserver.go:249] Found cloud.openshift.com token
I0331 03:52:08.535989       1 secretconfigobserver.go:119] support secret does not exist
I0331 03:52:08.539000       1 recorder.go:161] Pruning old reports every 6h48m20s, max age is 288h0m0s
I0331 03:52:08.543980       1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message=
I0331 03:52:08.543987       1 controllerstatus.go:80] name=insightsreport healthy=true reason= message=
I0331 03:52:08.543998       1 insightsreport.go:296] Starting report retriever
I0331 03:52:08.544000       1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s
I0331 03:52:08.544005       1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s
I0331 03:52:08.544014       1 periodic.go:209] Running clusterconfig gatherer
I0331 03:52:08.544053       1 tasks_processing.go:45] number of workers: 64
I0331 03:52:08.544080       1 tasks_processing.go:69] worker 3 listening for tasks.
I0331 03:52:08.544088       1 tasks_processing.go:69] worker 1 listening for tasks.
I0331 03:52:08.544090       1 tasks_processing.go:69] worker 2 listening for tasks.
I0331 03:52:08.544099       1 tasks_processing.go:69] worker 19 listening for tasks.
I0331 03:52:08.544101       1 tasks_processing.go:69] worker 0 listening for tasks.
I0331 03:52:08.544109       1 tasks_processing.go:71] worker 0 working on openstack_controlplanes task.
I0331 03:52:08.544117       1 tasks_processing.go:69] worker 5 listening for tasks.
I0331 03:52:08.544118       1 tasks_processing.go:69] worker 17 listening for tasks.
I0331 03:52:08.544121       1 tasks_processing.go:69] worker 15 listening for tasks.
I0331 03:52:08.544128       1 tasks_processing.go:69] worker 6 listening for tasks.
I0331 03:52:08.544116       1 tasks_processing.go:69] worker 20 listening for tasks.
I0331 03:52:08.544109       1 tasks_processing.go:69] worker 16 listening for tasks.
I0331 03:52:08.544108       1 tasks_processing.go:69] worker 4 listening for tasks.
I0331 03:52:08.544140       1 tasks_processing.go:69] worker 9 listening for tasks.
I0331 03:52:08.544125       1 tasks_processing.go:69] worker 44 listening for tasks.
I0331 03:52:08.544143       1 tasks_processing.go:69] worker 13 listening for tasks.
I0331 03:52:08.544147       1 tasks_processing.go:69] worker 10 listening for tasks.
I0331 03:52:08.544149       1 tasks_processing.go:69] worker 8 listening for tasks.
I0331 03:52:08.544132       1 tasks_processing.go:69] worker 45 listening for tasks.
I0331 03:52:08.544136       1 tasks_processing.go:69] worker 7 listening for tasks.
I0331 03:52:08.544159       1 tasks_processing.go:69] worker 12 listening for tasks.
I0331 03:52:08.544153       1 tasks_processing.go:69] worker 43 listening for tasks.
I0331 03:52:08.544159       1 tasks_processing.go:69] worker 57 listening for tasks.
I0331 03:52:08.544183       1 tasks_processing.go:69] worker 61 listening for tasks.
I0331 03:52:08.544147       1 tasks_processing.go:69] worker 47 listening for tasks.
I0331 03:52:08.544188       1 tasks_processing.go:71] worker 1 working on infrastructures task.
I0331 03:52:08.544194       1 tasks_processing.go:69] worker 21 listening for tasks.
I0331 03:52:08.544195       1 tasks_processing.go:69] worker 51 listening for tasks.
I0331 03:52:08.544199       1 tasks_processing.go:69] worker 56 listening for tasks.
I0331 03:52:08.544200       1 tasks_processing.go:69] worker 52 listening for tasks.
I0331 03:52:08.544197       1 tasks_processing.go:69] worker 59 listening for tasks.
I0331 03:52:08.544209       1 tasks_processing.go:69] worker 50 listening for tasks.
I0331 03:52:08.544195       1 tasks_processing.go:69] worker 55 listening for tasks.
I0331 03:52:08.544216       1 tasks_processing.go:69] worker 37 listening for tasks.
I0331 03:52:08.544220       1 tasks_processing.go:69] worker 35 listening for tasks.
I0331 03:52:08.544222       1 tasks_processing.go:71] worker 55 working on machine_healthchecks task.
I0331 03:52:08.544222       1 tasks_processing.go:69] worker 39 listening for tasks.
I0331 03:52:08.544226       1 tasks_processing.go:71] worker 21 working on mutating_webhook_configurations task.
I0331 03:52:08.544228       1 tasks_processing.go:71] worker 17 working on feature_gates task.
I0331 03:52:08.544231       1 tasks_processing.go:71] worker 7 working on jaegers task.
I0331 03:52:08.544233       1 tasks_processing.go:69] worker 25 listening for tasks.
I0331 03:52:08.544235       1 tasks_processing.go:71] worker 4 working on sap_config task.
I0331 03:52:08.544240       1 tasks_processing.go:69] worker 26 listening for tasks.
I0331 03:52:08.544244       1 tasks_processing.go:69] worker 29 listening for tasks.
I0331 03:52:08.544241       1 tasks_processing.go:71] worker 56 working on machines task.
I0331 03:52:08.544248       1 tasks_processing.go:71] worker 51 working on openstack_dataplanenodesets task.
I0331 03:52:08.544252       1 tasks_processing.go:69] worker 27 listening for tasks.
I0331 03:52:08.544139       1 tasks_processing.go:69] worker 46 listening for tasks.
I0331 03:52:08.544162       1 tasks_processing.go:69] worker 63 listening for tasks.
I0331 03:52:08.544164       1 tasks_processing.go:69] worker 60 listening for tasks.
I0331 03:52:08.544181       1 tasks_processing.go:69] worker 58 listening for tasks.
I0331 03:52:08.544180       1 tasks_processing.go:69] worker 49 listening for tasks.
I0331 03:52:08.544325       1 tasks_processing.go:71] worker 44 working on openstack_dataplanedeployments task.
I0331 03:52:08.544363       1 tasks_processing.go:69] worker 31 listening for tasks.
I0331 03:52:08.544216       1 tasks_processing.go:69] worker 23 listening for tasks.
I0331 03:52:08.544385       1 tasks_processing.go:71] worker 23 working on support_secret task.
I0331 03:52:08.544154       1 tasks_processing.go:69] worker 48 listening for tasks.
I0331 03:52:08.544208       1 tasks_processing.go:69] worker 22 listening for tasks.
I0331 03:52:08.544233       1 tasks_processing.go:69] worker 28 listening for tasks.
I0331 03:52:08.544165       1 tasks_processing.go:69] worker 14 listening for tasks.
I0331 03:52:08.544183       1 tasks_processing.go:71] worker 3 working on machine_config_pools task.
I0331 03:52:08.544109       1 tasks_processing.go:71] worker 19 working on authentication task.
I0331 03:52:08.544458       1 tasks_processing.go:71] worker 22 working on aggregated_monitoring_cr_names task.
I0331 03:52:08.544465       1 tasks_processing.go:71] worker 28 working on monitoring_persistent_volumes task.
I0331 03:52:08.544500       1 tasks_processing.go:71] worker 48 working on schedulers task.
I0331 03:52:08.544191       1 tasks_processing.go:69] worker 54 listening for tasks.
I0331 03:52:08.544562       1 tasks_processing.go:71] worker 54 working on certificate_signing_requests task.
I0331 03:52:08.544130       1 tasks_processing.go:69] worker 18 listening for tasks.
I0331 03:52:08.544201       1 tasks_processing.go:69] worker 32 listening for tasks.
I0331 03:52:08.544158       1 tasks_processing.go:69] worker 62 listening for tasks.
I0331 03:52:08.544101       1 tasks_processing.go:69] worker 11 listening for tasks.
I0331 03:52:08.544203       1 tasks_processing.go:69] worker 33 listening for tasks.
I0331 03:52:08.544198       1 tasks_processing.go:71] worker 2 working on machine_autoscalers task.
I0331 03:52:08.544191       1 tasks_processing.go:69] worker 53 listening for tasks.
I0331 03:52:08.544211       1 tasks_processing.go:69] worker 34 listening for tasks.
I0331 03:52:08.544215       1 tasks_processing.go:69] worker 40 listening for tasks.
I0331 03:52:08.544217       1 tasks_processing.go:71] worker 50 working on image task.
I0331 03:52:08.544456       1 tasks_processing.go:71] worker 14 working on ceph_cluster task.
I0331 03:52:08.544726       1 tasks_processing.go:71] worker 33 working on networks task.
I0331 03:52:08.544223       1 tasks_processing.go:71] worker 45 working on metrics task.
W0331 03:52:08.544873       1 gather_most_recent_metrics.go:64] Unable to load metrics client, no metrics will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory
I0331 03:52:08.544893       1 tasks_processing.go:71] worker 45 working on image_pruners task.
I0331 03:52:08.544223       1 tasks_processing.go:69] worker 41 listening for tasks.
I0331 03:52:08.544224       1 tasks_processing.go:69] worker 24 listening for tasks.
I0331 03:52:08.544227       1 tasks_processing.go:71] worker 47 working on version task.
I0331 03:52:08.544928       1 gather.go:177] gatherer "clusterconfig" function "metrics" took 42.98µs to process 0 records
I0331 03:52:08.544941       1 tasks_processing.go:71] worker 41 working on container_images task.
I0331 03:52:08.544938       1 tasks_processing.go:71] worker 24 working on service_accounts task.
I0331 03:52:08.544216       1 tasks_processing.go:69] worker 36 listening for tasks.
I0331 03:52:08.545052       1 tasks_processing.go:71] worker 36 working on nodenetworkstates task.
I0331 03:52:08.544231       1 tasks_processing.go:69] worker 42 listening for tasks.
I0331 03:52:08.544305       1 tasks_processing.go:71] worker 12 working on config_maps task.
I0331 03:52:08.544305       1 tasks_processing.go:71] worker 20 working on pdbs task.
I0331 03:52:08.545270       1 tasks_processing.go:71] worker 42 working on silenced_alerts task.
I0331 03:52:08.544309       1 tasks_processing.go:71] worker 43 working on overlapping_namespace_uids task.
W0331 03:52:08.545301       1 gather_silenced_alerts.go:38] Unable to load alerts client, no alerts will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory
I0331 03:52:08.545314       1 tasks_processing.go:71] worker 42 working on active_alerts task.
W0331 03:52:08.545332       1 gather_active_alerts.go:54] Unable to load alerts client, no alerts will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory
I0331 03:52:08.544309       1 tasks_processing.go:71] worker 16 working on operators task.
I0331 03:52:08.545384       1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 29.523µs to process 0 records
I0331 03:52:08.545403       1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 18.468µs to process 0 records
I0331 03:52:08.545416       1 tasks_processing.go:71] worker 42 working on olm_operators task.
I0331 03:52:08.544312       1 tasks_processing.go:71] worker 57 working on sap_pods task.
I0331 03:52:08.544315       1 tasks_processing.go:71] worker 52 working on openstack_version task.
I0331 03:52:08.544316       1 tasks_processing.go:71] worker 61 working on qemu_kubevirt_launcher_logs task.
I0331 03:52:08.544316       1 tasks_processing.go:71] worker 15 working on storage_cluster task.
I0331 03:52:08.544322       1 tasks_processing.go:71] worker 13 working on machine_configs task.
I0331 03:52:08.544322       1 tasks_processing.go:71] worker 9 working on container_runtime_configs task.
I0331 03:52:08.544328       1 tasks_processing.go:71] worker 59 working on operators_pods_and_events task.
I0331 03:52:08.544333       1 tasks_processing.go:71] worker 49 working on crds task.
I0331 03:52:08.544336       1 tasks_processing.go:71] worker 29 working on cluster_apiserver task.
I0331 03:52:08.544337       1 tasks_processing.go:71] worker 27 working on openshift_logging task.
I0331 03:52:08.544335       1 tasks_processing.go:71] worker 10 working on number_of_pods_and_netnamespaces_with_sdn_annotations task.
I0331 03:52:08.544339       1 tasks_processing.go:71] worker 37 working on clusterroles task.
I0331 03:52:08.544341       1 tasks_processing.go:71] worker 46 working on storage_classes task.
I0331 03:52:08.544343       1 tasks_processing.go:71] worker 25 working on ingress task.
I0331 03:52:08.544343       1 tasks_processing.go:71] worker 39 working on dvo_metrics task.
I0331 03:52:08.544344       1 tasks_processing.go:71] worker 63 working on ingress_certificates task.
I0331 03:52:08.544348       1 tasks_processing.go:71] worker 60 working on pod_network_connectivity_checks task.
I0331 03:52:08.544346       1 tasks_processing.go:71] worker 26 working on nodes task.
I0331 03:52:08.544348       1 tasks_processing.go:71] worker 35 working on sap_datahubs task.
I0331 03:52:08.544351       1 tasks_processing.go:71] worker 58 working on lokistack task.
I0331 03:52:08.544353       1 tasks_processing.go:71] worker 8 working on openshift_machine_api_events task.
I0331 03:52:08.544360       1 tasks_processing.go:69] worker 30 listening for tasks.
I0331 03:52:08.544377       1 tasks_processing.go:71] worker 31 working on image_registries task.
I0331 03:52:08.544210       1 tasks_processing.go:69] worker 38 listening for tasks.
I0331 03:52:08.544699       1 tasks_processing.go:71] worker 11 working on tsdb_status task.
W0331 03:52:08.545556       1 gather_prometheus_tsdb_status.go:38] Unable to load metrics client, tsdb status cannot be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory
I0331 03:52:08.545588       1 tasks_processing.go:74] worker 11 stopped.
I0331 03:52:08.545606       1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 55.406µs to process 0 records
I0331 03:52:08.545875       1 tasks_processing.go:74] worker 30 stopped.
I0331 03:52:08.544306       1 tasks_processing.go:71] worker 6 working on node_logs task.
I0331 03:52:08.544707       1 tasks_processing.go:71] worker 32 working on cost_management_metrics_configs task.
I0331 03:52:08.544703       1 tasks_processing.go:71] worker 18 working on install_plans task.
I0331 03:52:08.544711       1 tasks_processing.go:71] worker 62 working on machine_sets task.
I0331 03:52:08.548880       1 tasks_processing.go:74] worker 38 stopped.
I0331 03:52:08.544700       1 tasks_processing.go:71] worker 40 working on validating_webhook_configurations task.
I0331 03:52:08.544716       1 tasks_processing.go:71] worker 53 working on nodenetworkconfigurationpolicies task.
I0331 03:52:08.544222       1 tasks_processing.go:71] worker 5 working on proxies task.
I0331 03:52:08.544718       1 tasks_processing.go:71] worker 34 working on oauths task.
I0331 03:52:08.549650       1 tasks_processing.go:74] worker 56 stopped.
E0331 03:52:08.549687       1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope
I0331 03:52:08.549712       1 gather.go:177] gatherer "clusterconfig" function "machines" took 5.392846ms to process 0 records
I0331 03:52:08.550563       1 tasks_processing.go:74] worker 44 stopped.
I0331 03:52:08.550577       1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 6.192335ms to process 0 records
I0331 03:52:08.550585       1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 6.450936ms to process 0 records
I0331 03:52:08.550589       1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 6.318931ms to process 0 records
I0331 03:52:08.550594       1 tasks_processing.go:74] worker 51 stopped.
I0331 03:52:08.550597       1 tasks_processing.go:74] worker 0 stopped.
I0331 03:52:08.550605       1 tasks_processing.go:74] worker 55 stopped.
E0331 03:52:08.550620       1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope
I0331 03:52:08.550631       1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 6.373132ms to process 0 records
I0331 03:52:08.554818       1 tasks_processing.go:74] worker 4 stopped.
I0331 03:52:08.554827       1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 10.568314ms to process 0 records
I0331 03:52:08.558471       1 controller.go:128] Initializing last reported time to 0001-01-01T00:00:00Z
I0331 03:52:08.558491       1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready
I0331 03:52:08.558498       1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready
I0331 03:52:08.558502       1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready
I0331 03:52:08.558519       1 controller.go:489] The operator is still being initialized
I0331 03:52:08.558527       1 controller.go:512] The operator is healthy
I0331 03:52:08.566071       1 tasks_processing.go:74] worker 2 stopped.
I0331 03:52:08.566088       1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 21.479709ms to process 0 records
I0331 03:52:08.566097       1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 21.831852ms to process 0 records
I0331 03:52:08.566103       1 tasks_processing.go:74] worker 7 stopped.
I0331 03:52:08.566186       1 tasks_processing.go:74] worker 57 stopped.
I0331 03:52:08.566204       1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 20.146019ms to process 0 records
I0331 03:52:08.566265       1 tasks_processing.go:74] worker 1 stopped.
I0331 03:52:08.566969       1 recorder.go:75] Recording config/infrastructure with fingerprint=9c76a82d3dc60076698b7a9ea931797e57c667c65eb187248ab9deaff2b459f3
I0331 03:52:08.566989       1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 22.056329ms to process 1 records
I0331 03:52:08.567089       1 tasks_processing.go:74] worker 17 stopped.
I0331 03:52:08.567146       1 recorder.go:75] Recording config/featuregate with fingerprint=ace966eca666ec45f2108c17902b99aac0b2b350eca61b22abdfc44e97b9c9b6
I0331 03:52:08.567159       1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 22.130676ms to process 1 records
I0331 03:52:08.592695       1 tasks_processing.go:74] worker 3 stopped.
I0331 03:52:08.592714       1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 48.285742ms to process 0 records
I0331 03:52:08.592767       1 tasks_processing.go:74] worker 28 stopped.
I0331 03:52:08.592791       1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 48.284931ms to process 0 records
I0331 03:52:08.592920       1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=19fa7d4dd241a396c2ff2c45839294b116a4b5fbc385d791f3eb2e3e7bc19f05
I0331 03:52:08.592923       1 tasks_processing.go:74] worker 48 stopped.
I0331 03:52:08.592938       1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 48.251387ms to process 1 records
I0331 03:52:08.592946       1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 46.910485ms to process 0 records
I0331 03:52:08.592954       1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 47.786528ms to process 0 records
I0331 03:52:08.592959       1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 47.347621ms to process 0 records
I0331 03:52:08.592965       1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 46.313774ms to process 0 records
I0331 03:52:08.592972       1 tasks_processing.go:74] worker 52 stopped.
I0331 03:52:08.592979       1 tasks_processing.go:74] worker 35 stopped.
I0331 03:52:08.592984       1 tasks_processing.go:74] worker 36 stopped.
I0331 03:52:08.592983       1 tasks_processing.go:74] worker 50 stopped.
I0331 03:52:08.592989       1 tasks_processing.go:74] worker 27 stopped.
I0331 03:52:08.593131       1 recorder.go:75] Recording config/image with fingerprint=e9314b46c3cd288bed0afadbc0949eb4fe9f823bdc16dbe8218241d70d782e80
I0331 03:52:08.593154       1 gather.go:177] gatherer "clusterconfig" function "image" took 48.342445ms to process 1 records
I0331 03:52:08.593384       1 tasks_processing.go:74] worker 19 stopped.
I0331 03:52:08.593531       1 recorder.go:75] Recording config/authentication with fingerprint=903d5c2edb64929e893d89fd4ccffc1d83436c56c41038ec28b482ba21a084cf
I0331 03:52:08.593549       1 gather.go:177] gatherer "clusterconfig" function "authentication" took 48.622106ms to process 1 records
I0331 03:52:08.593562       1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 46.860915ms to process 0 records
I0331 03:52:08.593582       1 tasks_processing.go:74] worker 58 stopped.
I0331 03:52:08.593653       1 tasks_processing.go:74] worker 21 stopped.
I0331 03:52:08.593750       1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=38b8933d3ee1772ccaa0efe7fe81e4e358a392e13a1b742cfd438e2cb18baf40
I0331 03:52:08.593831       1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=a8d058a26d05754b38d1e3437875e135ba181e8dee2d84424abd4171275b4d01
I0331 03:52:08.593926       1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=7c2382866f98dceaa8fe388355dc5c8230de64b01fac36bcf0b7e6285c55c7af
I0331 03:52:08.593964       1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 48.888152ms to process 3 records
I0331 03:52:08.593995       1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 48.569187ms to process 0 records
E0331 03:52:08.594025       1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found
I0331 03:52:08.594034       1 tasks_processing.go:74] worker 14 stopped.
I0331 03:52:08.594040       1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 48.914392ms to process 0 records
I0331 03:52:08.594047       1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 47.078792ms to process 0 records
I0331 03:52:08.594055       1 tasks_processing.go:74] worker 9 stopped.
I0331 03:52:08.594057       1 tasks_processing.go:74] worker 23 stopped.
I0331 03:52:08.594136       1 tasks_processing.go:74] worker 33 stopped.
I0331 03:52:08.594281       1 recorder.go:75] Recording config/network with fingerprint=a24b48cc383a37c6e9802c9dc2009c2ca43a476efd2676b52232e1f7c8ebd6c6
I0331 03:52:08.594305       1 gather.go:177] gatherer "clusterconfig" function "networks" took 48.619812ms to process 1 records
E0331 03:52:08.594316       1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io)
I0331 03:52:08.594324       1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 46.7882ms to process 0 records
I0331 03:52:08.594331       1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 46.670561ms to process 0 records
I0331 03:52:08.594335       1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 48.134418ms to process 0 records
I0331 03:52:08.594339       1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 45.457254ms to process 0 records
I0331 03:52:08.594344       1 tasks_processing.go:74] worker 32 stopped.
I0331 03:52:08.594356       1 tasks_processing.go:74] worker 60 stopped.
I0331 03:52:08.594358       1 tasks_processing.go:74] worker 42 stopped.
I0331 03:52:08.594364       1 tasks_processing.go:74] worker 62 stopped.
I0331 03:52:08.594451       1 tasks_processing.go:74] worker 45 stopped.
I0331 03:52:08.594545       1 gather_logs.go:145] no pods in  namespace were found
I0331 03:52:08.594590       1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=374d682db87e5b4abeff8c0be615c974ddd4652f5507f63d5d30655d6851ab40
I0331 03:52:08.594598       1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 48.906265ms to process 1 records
I0331 03:52:08.594670       1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=0b536f92e2a92aff4b011db280f9a558286d2b501df4cfbc7944340392f301ea
I0331 03:52:08.594692       1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=fd72bdcb4d68d2d9d3241cd6bf2bf11eb553de0903e5972ed1667cecacec96ec
I0331 03:52:08.594700       1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 46.463313ms to process 2 records
I0331 03:52:08.594706       1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 47.140885ms to process 0 records
I0331 03:52:08.594672       1 tasks_processing.go:74] worker 46 stopped.
I0331 03:52:08.594778       1 tasks_processing.go:74] worker 15 stopped.
I0331 03:52:08.594904       1 recorder.go:75] Recording config/apiserver with fingerprint=fef7bcd6ec1c2e432dcb73cab518872e196c9d6d32d965f53fce5c50c03a6056
I0331 03:52:08.594931       1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 47.43085ms to process 1 records
I0331 03:52:08.594944       1 tasks_processing.go:74] worker 29 stopped.
I0331 03:52:08.595059       1 tasks_processing.go:74] worker 25 stopped.
I0331 03:52:08.595242       1 recorder.go:75] Recording config/ingress with fingerprint=eea365db3992c07d6869f31b3bd08b566cfc4c0649ee9f5f001f0402cf251c74
I0331 03:52:08.595265       1 gather.go:177] gatherer "clusterconfig" function "ingress" took 46.411983ms to process 1 records
I0331 03:52:08.595277       1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 47.959843ms to process 0 records
I0331 03:52:08.595392       1 tasks_processing.go:74] worker 61 stopped.
I0331 03:52:08.595416       1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=31466ed2407dfbd8fce4ae0f71af21fa0a544d5958eba97b2e6704dd447e3e1a
I0331 03:52:08.595485       1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=84a60c2d1e4f438a5318716941374f77a297d2670d42c175fd02973ec8d76937
I0331 03:52:08.595512       1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=fe9cc210c47be5865cc343f93328902627b6b19a2a2e14cc2dbc4a53907b1229
I0331 03:52:08.595521       1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 49.456922ms to process 3 records
I0331 03:52:08.595533       1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 45.84826ms to process 0 records
I0331 03:52:08.595569       1 tasks_processing.go:74] worker 20 stopped.
I0331 03:52:08.595583       1 tasks_processing.go:74] worker 53 stopped.
I0331 03:52:08.595688       1 tasks_processing.go:74] worker 5 stopped.
I0331 03:52:08.595760       1 recorder.go:75] Recording config/proxy with fingerprint=2c4ef8123e90f84f8a082e656b2f6b665eb4a87d94248b9557cce7f6b4c44372
I0331 03:52:08.595780       1 gather.go:177] gatherer "clusterconfig" function "proxies" took 45.793971ms to process 1 records
I0331 03:52:08.595943       1 tasks_processing.go:74] worker 34 stopped.
I0331 03:52:08.596972       1 recorder.go:75] Recording config/oauth with fingerprint=4af6c3df3c1125441933fb1a58a0737106bb4b878c9ff4eb179d3bb4cb22e2e1
I0331 03:52:08.597056       1 gather.go:177] gatherer "clusterconfig" function "oauths" took 46.422318ms to process 1 records
W0331 03:52:08.601965       1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again.
I0331 03:52:08.602684       1 tasks_processing.go:74] worker 8 stopped.
I0331 03:52:08.602702       1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 56.410533ms to process 0 records
I0331 03:52:08.602878       1 tasks_processing.go:74] worker 6 stopped.
I0331 03:52:08.602891       1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 56.932226ms to process 0 records
I0331 03:52:08.603289       1 tasks_processing.go:74] worker 40 stopped.
I0331 03:52:08.603451       1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=d7f3b12951793c674acbfcd68e259312d9b7d008c3a6953962f40eba34d08b38
I0331 03:52:08.603651       1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=c8ea30c3966244307772b25f40649bfc6593b741c7409915e8146ffdd8fba6fe
I0331 03:52:08.603716       1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=521789f9e0609450303ea06cfbdc90a63fd0ac3cfb258d0d79eae012c178271e
I0331 03:52:08.603781       1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=0cd4706a66be7338283744150085f4c0a90ea5f00199cc518ab7c38b94dc5df0
I0331 03:52:08.603854       1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=5f10923784b1300d1427f8b36282de17996bfea4577209f80d07a78c2b2cb5a7
I0331 03:52:08.603899       1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=7f50085787ebcb9960241c4ed62841da869f2372ead4d39682bccb7143b2444c
I0331 03:52:08.603943       1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=c96b09a790822dbe870b56cb71ace539540b194bf95fdec9a039c1f7bb677c3b
I0331 03:52:08.604031       1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=25eea9d856f826e10b133593738074416c2ca84b8dc532b234fef98ecc1bd2ef
I0331 03:52:08.604094       1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=f3284003ed0963ef61e2e0d5393885e292738883a4e2a341c7f69fbee73fec41
I0331 03:52:08.604158       1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=278fff0bca80fb2a1ca528b34e15c74ded4807e652bfc724c6751756e3c2cf4b
I0331 03:52:08.604228       1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=08a6536949032a35ddfa632946924ace692713e309f61748f8a8df6be53afda7
I0331 03:52:08.604243       1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 54.369377ms to process 11 records
I0331 03:52:08.604415       1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s
I0331 03:52:08.604605       1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s
I0331 03:52:08.604686       1 recorder.go:75] Recording config/node/ip-10-0-0-103.ec2.internal with fingerprint=bde8145543557fb784db2e3891c4b8767cd466292d9d3b99ee1ba428126f29d0
I0331 03:52:08.604704       1 gather.go:177] gatherer "clusterconfig" function "nodes" took 57.858672ms to process 1 records
I0331 03:52:08.604747       1 tasks_processing.go:74] worker 26 stopped.
W0331 03:52:08.604896       1 operator.go:288] started
I0331 03:52:08.605089       1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer
I0331 03:52:08.605185       1 tasks_processing.go:74] worker 31 stopped.
I0331 03:52:08.605307       1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=79c923f3c8e980b90709fac3f4e0ed119aae68ad61cc6277a45f6f6f9df9a855
I0331 03:52:08.605321       1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 56.740015ms to process 1 records
I0331 03:52:08.605329       1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 59.581769ms to process 0 records
I0331 03:52:08.605339       1 tasks_processing.go:74] worker 54 stopped.
I0331 03:52:08.605782       1 tasks_processing.go:74] worker 43 stopped.
I0331 03:52:08.605809       1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945
I0331 03:52:08.605820       1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 60.461507ms to process 1 records
I0331 03:52:08.609980       1 tasks_processing.go:74] worker 41 stopped.
I0331 03:52:08.610054       1 recorder.go:75] Recording config/running_containers with fingerprint=171b41814f6dcf3a8dd239959f8aa6d9e9c47c34cac11eb41ef1af082a83a901
I0331 03:52:08.610069       1 gather.go:177] gatherer "clusterconfig" function "container_images" took 65.023413ms to process 1 records
I0331 03:52:08.612610       1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file
I0331 03:52:08.612618       1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController
I0331 03:52:08.612679       1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file
I0331 03:52:08.616870       1 base_controller.go:82] Caches are synced for ConfigController 
I0331 03:52:08.616883       1 base_controller.go:119] Starting #1 worker of ConfigController controller ...
I0331 03:52:08.622474       1 tasks_processing.go:74] worker 37 stopped.
I0331 03:52:08.622666       1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=da4e10e41a33a822e8771d853c5ebcf89340b731daf808b3aab85484e3b21219
I0331 03:52:08.622816       1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=d4550924ddef0b1bdc93181b3cd97c10a04cfccd68b115ec0fd5735e24446469
I0331 03:52:08.622830       1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 76.372485ms to process 2 records
I0331 03:52:08.626833       1 configmapobserver.go:84] configmaps "insights-config" not found
I0331 03:52:08.626943       1 tasks_processing.go:74] worker 10 stopped.
I0331 03:52:08.626962       1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 81.321167ms to process 0 records
I0331 03:52:08.627054       1 tasks_processing.go:74] worker 49 stopped.
I0331 03:52:08.627373       1 prometheus_rules.go:88] Prometheus rules successfully created
I0331 03:52:08.627449       1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready
I0331 03:52:08.627468       1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready
I0331 03:52:08.627474       1 controller.go:212] Source scaController *sca.Controller is not ready
I0331 03:52:08.627478       1 controller.go:212] Source clusterTransferController *clustertransfer.Controller is not ready
I0331 03:52:08.627482       1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready
I0331 03:52:08.627506       1 controller.go:489] The operator is still being initialized
I0331 03:52:08.627514       1 controller.go:512] The operator is healthy
I0331 03:52:08.627590       1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=a69b50a34e387e18e0c9903dc1b6eed7ed2f2e44cc75b495f01261f328e08b73
I0331 03:52:08.627847       1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=683b354c7e963cb8dc5fa7287c6638ee82bb6ffe9c159e1d49773753e2a23efd
I0331 03:52:08.627861       1 gather.go:177] gatherer "clusterconfig" function "crds" took 80.173717ms to process 2 records
I0331 03:52:08.627872       1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 82.795172ms to process 0 records
I0331 03:52:08.627878       1 tasks_processing.go:74] worker 22 stopped.
E0331 03:52:08.636041       1 cluster_transfer.go:95] failed to pull cluster transfer: unable to retrieve cluster transfer data from https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/: Get "https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/?search=cluster_uuid+is+%27ee0a9d92-1926-487c-908a-2ec17992f53d%27+and+status+is+%27accepted%27": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.12:50005->172.30.0.10:53: read: connection refused
I0331 03:52:08.636054       1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=Disconnected message=failed to pull cluster transfer: unable to retrieve cluster transfer data from https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/: Get "https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/?search=cluster_uuid+is+%27ee0a9d92-1926-487c-908a-2ec17992f53d%27+and+status+is+%27accepted%27": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.12:50005->172.30.0.10:53: read: connection refused
I0331 03:52:08.650909       1 tasks_processing.go:74] worker 63 stopped.
E0331 03:52:08.650925       1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found
E0331 03:52:08.650932       1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2pcf5093fb1dpqq37df3mvn2eh63vsdj-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2pcf5093fb1dpqq37df3mvn2eh63vsdj-primary-cert-bundle-secret" not found
I0331 03:52:08.650992       1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=c5bb667fdb0382ad1fc25a8d0c49d2ba09d7e8f07fa7864dcc01c162073ffa9c
I0331 03:52:08.651005       1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 104.33691ms to process 1 records
I0331 03:52:08.659314       1 tasks_processing.go:74] worker 47 stopped.
I0331 03:52:08.659763       1 recorder.go:75] Recording config/version with fingerprint=d1ed44320db96278d8a97f09303c8895f1d4544a91c6717efe3e9bbf6d33d554
I0331 03:52:08.659775       1 recorder.go:75] Recording config/id with fingerprint=248ca49c4be3d64130c8fffb701a542cc1750b5e3614f0e126e8fb8ffe964c05
I0331 03:52:08.659804       1 gather.go:177] gatherer "clusterconfig" function "version" took 114.376822ms to process 2 records
I0331 03:52:08.659949       1 tasks_processing.go:74] worker 12 stopped.
E0331 03:52:08.659966       1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found
E0331 03:52:08.659975       1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found
E0331 03:52:08.659980       1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found
I0331 03:52:08.659992       1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=26b6661162b099a0f5a279859b4f46c867929a79d9a4a41fde4be4e6fe138018
I0331 03:52:08.660021       1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=2b7e7d968c1ca0e033f2ae1514829fefa6008cf7d89addbe266d351cfb9cca19
I0331 03:52:08.660030       1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=c93090eb0d2a4736885abeb79c91680cfd01fda46464f83456b085d4dc8239f0
I0331 03:52:08.660037       1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a
I0331 03:52:08.660043       1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
I0331 03:52:08.660105       1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0
I0331 03:52:08.660116       1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d
I0331 03:52:08.660124       1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 114.871101ms to process 7 records
I0331 03:52:08.675617       1 tasks_processing.go:74] worker 13 stopped.
I0331 03:52:08.675656       1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499
I0331 03:52:08.675702       1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 130.033986ms to process 1 records
I0331 03:52:08.687515       1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload
W0331 03:52:08.692339       1 sca.go:161] Failed to pull SCA certs from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: unable to retrieve SCA certs data from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: Post "https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.12:38364->172.30.0.10:53: read: connection refused
I0331 03:52:08.692352       1 controllerstatus.go:80] name=scaController healthy=true reason=NonHTTPError message=Failed to pull SCA certs from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: unable to retrieve SCA certs data from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: Post "https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.12:38364->172.30.0.10:53: read: connection refused
I0331 03:52:08.705748       1 base_controller.go:82] Caches are synced for LoggingSyncer 
I0331 03:52:08.705766       1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ...
I0331 03:52:08.713641       1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found
I0331 03:52:08.724571       1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found
W0331 03:52:09.599111       1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again.
I0331 03:52:09.814130       1 gather_cluster_operator_pods_and_events.go:121] Found 9 pods with 11 containers
I0331 03:52:09.814144       1 gather_cluster_operator_pods_and_events.go:235] Maximum buffer size: 2287802 bytes
I0331 03:52:09.815590       1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns container dns-default-vlcm7 pod in namespace openshift-dns (previous: false).
I0331 03:52:10.046533       1 gather_cluster_operators.go:184] Unable to get configs.samples.operator.openshift.io resource due to: configs.samples.operator.openshift.io "cluster" not found
I0331 03:52:10.051482       1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-vlcm7 pod in namespace openshift-dns for failing operator dns (previous: false): "container \"dns\" in pod \"dns-default-vlcm7\" is waiting to start: ContainerCreating"
I0331 03:52:10.051499       1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"dns\" in pod \"dns-default-vlcm7\" is waiting to start: ContainerCreating"
I0331 03:52:10.051506       1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-rbac-proxy container dns-default-vlcm7 pod in namespace openshift-dns (previous: false).
I0331 03:52:10.226520       1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-vlcm7 pod in namespace openshift-dns for failing operator kube-rbac-proxy (previous: false): "container \"kube-rbac-proxy\" in pod \"dns-default-vlcm7\" is waiting to start: ContainerCreating"
I0331 03:52:10.226540       1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"kube-rbac-proxy\" in pod \"dns-default-vlcm7\" is waiting to start: ContainerCreating"
I0331 03:52:10.226550       1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-7bjnr pod in namespace openshift-dns (previous: false).
I0331 03:52:10.418389       1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty"
I0331 03:52:10.418410       1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-56f5f6f959-klwlv pod in namespace openshift-image-registry (previous: false).
I0331 03:52:10.447467       1 tasks_processing.go:74] worker 16 stopped.
I0331 03:52:10.447518       1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=f200bb91ac8542f575d98779e95c2e72e4721684101912c2e90bc4d33f95a8ec
I0331 03:52:10.447547       1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=31a2bd554986abb811ed8fdb12bfffc470f4f0681a3c86ab1054f983b0219fe9
I0331 03:52:10.447580       1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e
I0331 03:52:10.447606       1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=afeff27ba91944f7a83c13e0b15ded84851ccf8b83f2944a96663cbbea894eb0
I0331 03:52:10.447623       1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f
I0331 03:52:10.447647       1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=be343fdc823375ba0aa662f236af8789514d24332f6713988b462a35c2e3c4d4
I0331 03:52:10.447682       1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=ee95042522bd5e972d1174c19daad2779494b1b9d96f14c33e7e92b1a4f21bce
I0331 03:52:10.447708       1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=5493ff81eb2fbb9d8b6c3f263f65b4687eed0a126f232b1aa14885593a30eb2c
I0331 03:52:10.447742       1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=463c7b75cb7d89b3f35236002c9bdacc84940d034744a2865333073d67c45812
I0331 03:52:10.447761       1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=0c0be9587b6128de224b6674c39a41e7dcdfa6afc25dda5ef8d4ab4b38458545
I0331 03:52:10.447770       1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762
I0331 03:52:10.447787       1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=6d59d19c6367f4c221ff59c2ea27f01ec44568d409a71d5d8dfb467c746c1c1a
I0331 03:52:10.447797       1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11
I0331 03:52:10.447820       1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=740531e0130fb69fc4b9fe00900442a4a7cb6a4d8934f145538a9534cfb3e43b
I0331 03:52:10.447830       1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a
I0331 03:52:10.447849       1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=e31cae4b8be4599312b72c337f548925de4a462b8747cdf180da317451792329
I0331 03:52:10.447858       1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db
I0331 03:52:10.447873       1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=8260b9d009629ef5c2e0ea3d1ccdfe3a6a0cab0e018c154d194e360551d4a659
I0331 03:52:10.447999       1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=695e165bfec2880ef69f7c5b38bc1999ef7c458906e95ae83c7c4c9a0b267dcc
I0331 03:52:10.448010       1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab
I0331 03:52:10.448017       1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d
I0331 03:52:10.448043       1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75
I0331 03:52:10.448066       1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=bdacf9856d00a8fc7b08728cc651e084f5d7ade9463d500a6f3139547b22f767
I0331 03:52:10.448090       1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=59b6d9ea7c62e2d254c8d3ba771273b7dbea0bd15726aa8bf4004f6d5be7a9cd
I0331 03:52:10.448099       1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76
I0331 03:52:10.448119       1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=561ffe0fefb80f2854893a08fa1a7850ce97d466696ab12da72ca1d389ed5c8b
I0331 03:52:10.448130       1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7
I0331 03:52:10.448144       1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=5ad5b5776da123726c01e0dc903d1c3c1ab23a040ddef663bdab1ac43a97206d
I0331 03:52:10.448158       1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=4998a233c079ae299a8e771e65fae6e83bc246d856c73945591854aacfacf3ad
I0331 03:52:10.448190       1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=5a89a3096e80131dc8b541817cb83429c21fa11b3fd775c36d3f19b590847368
I0331 03:52:10.448208       1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=2c4d49b1a329c503b782f9707684d1e104823f871e59f9ce8adc0e0030c8e3ca
I0331 03:52:10.448219       1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=06bf356ac9b4fcb1fdbe64198d9cd1363a10c86c6a5eb2203a875ee1913fa96e
I0331 03:52:10.448245       1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=fe47417eba9668aa172e95b9504ad3b8d229e07d6c17577274f53047ab99edb6
I0331 03:52:10.448266       1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e
I0331 03:52:10.448275       1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad
I0331 03:52:10.448282       1 gather.go:177] gatherer "clusterconfig" function "operators" took 1.902116248s to process 35 records
W0331 03:52:10.598946       1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again.
I0331 03:52:10.612200       1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty"
I0331 03:52:10.612218       1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-78f86fc49b-f9tx2 pod in namespace openshift-image-registry (previous: false).
I0331 03:52:10.812632       1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty"
I0331 03:52:10.812650       1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-78f86fc49b-fknx9 pod in namespace openshift-image-registry (previous: false).
I0331 03:52:11.018826       1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for image-registry-78f86fc49b-fknx9 pod in namespace openshift-image-registry for failing operator registry (previous: false): "container \"registry\" in pod \"image-registry-78f86fc49b-fknx9\" is waiting to start: ContainerCreating"
I0331 03:52:11.018845       1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"registry\" in pod \"image-registry-78f86fc49b-fknx9\" is waiting to start: ContainerCreating"
I0331 03:52:11.018863       1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-hhzfz pod in namespace openshift-image-registry (previous: false).
I0331 03:52:11.218661       1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty"
I0331 03:52:11.218680       1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-rhqvm pod in namespace openshift-ingress-canary (previous: false).
I0331 03:52:11.424080       1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for ingress-canary-rhqvm pod in namespace openshift-ingress-canary for failing operator serve-healthcheck-canary (previous: false): "container \"serve-healthcheck-canary\" in pod \"ingress-canary-rhqvm\" is waiting to start: ContainerCreating"
I0331 03:52:11.424098       1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"serve-healthcheck-canary\" in pod \"ingress-canary-rhqvm\" is waiting to start: ContainerCreating"
I0331 03:52:11.424114       1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for migrator container migrator-7d5f866c57-c47jw pod in namespace openshift-kube-storage-version-migrator (previous: false).
W0331 03:52:11.599071       1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again.
I0331 03:52:11.621140       1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for migrator-7d5f866c57-c47jw pod in namespace openshift-kube-storage-version-migrator for failing operator migrator (previous: false): "container \"migrator\" in pod \"migrator-7d5f866c57-c47jw\" is waiting to start: ContainerCreating"
I0331 03:52:11.621153       1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"migrator\" in pod \"migrator-7d5f866c57-c47jw\" is waiting to start: ContainerCreating"
I0331 03:52:11.621184       1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for graceful-termination container migrator-7d5f866c57-c47jw pod in namespace openshift-kube-storage-version-migrator (previous: false).
I0331 03:52:11.819466       1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-storage-version-migrator-operator container kube-storage-version-migrator-operator-74848b4cb9-pr4vj pod in namespace openshift-kube-storage-version-migrator-operator (previous: false).
I0331 03:52:12.021998       1 tasks_processing.go:74] worker 59 stopped.
I0331 03:52:12.022077       1 recorder.go:75] Recording events/openshift-dns with fingerprint=3528b303246e912625714d53554b4e11b6e9ceb5d12cece04d49cf88d4a19c29
I0331 03:52:12.022154       1 recorder.go:75] Recording events/openshift-image-registry with fingerprint=5a56ca17cc8bba84b2e0b0294162d1c44a5dd08ad7e9622a362a343e83f23e97
I0331 03:52:12.022194       1 recorder.go:75] Recording events/openshift-ingress-operator with fingerprint=04deb1d8a05af757d15ec69bc4f823019d28e4f933dfe5b40a5d1ba4ab4bb0d1
I0331 03:52:12.022221       1 recorder.go:75] Recording events/openshift-ingress with fingerprint=845a4411c87f2c600de1f8d3130dabf03ac86e8d33ad2b24efeeb0ad06bb66f5
I0331 03:52:12.022232       1 recorder.go:75] Recording events/openshift-ingress-canary with fingerprint=1002ccc28d6bee10e8e032abbde7df4d9d6e48d97c0ac8863c605b827ace05dc
I0331 03:52:12.022244       1 recorder.go:75] Recording events/openshift-kube-storage-version-migrator with fingerprint=90cf4001cff530255caec93a8ebd8011023bf25004052096423acf1bd909dd17
I0331 03:52:12.022290       1 recorder.go:75] Recording events/openshift-kube-storage-version-migrator-operator with fingerprint=3cead974edad16737cc00ffaa47fc0d2428bb0ac8b9f065be4a6350214fd12bd
I0331 03:52:12.022298       1 recorder.go:75] Recording config/pod/openshift-kube-storage-version-migrator/logs/migrator-7d5f866c57-c47jw/graceful-termination_current.log with fingerprint=860adfe07c0429092a11ac38db1420793bb3b6a3f7ddfea7205e01212ce43d89
I0331 03:52:12.022370       1 recorder.go:75] Recording config/pod/openshift-kube-storage-version-migrator-operator/logs/kube-storage-version-migrator-operator-74848b4cb9-pr4vj/kube-storage-version-migrator-operator_current.log with fingerprint=dc62f0dc5e8aa372264f9506e6b6e8be54345d29af4eb53d85b863a29bcbaf04
I0331 03:52:12.022379       1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 3.475486354s to process 9 records
W0331 03:52:12.599357       1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again.
W0331 03:52:13.595197       1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded
I0331 03:52:13.595239       1 tasks_processing.go:74] worker 39 stopped.
E0331 03:52:13.595264       1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded
I0331 03:52:13.595288       1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
W0331 03:52:13.595314       1 gather.go:155] issue recording gatherer "clusterconfig" function "dvo_metrics" result "config/dvo_metrics" because of the warning: warning: the record with the same fingerprint "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" was already recorded at path "config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt", recording another one with a different path "config/dvo_metrics"
I0331 03:52:13.595342       1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.048831716s to process 1 records
I0331 03:52:17.446688       1 configmapobserver.go:84] configmaps "insights-config" not found
I0331 03:52:21.213769       1 tasks_processing.go:74] worker 18 stopped.
I0331 03:52:21.213803       1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b
I0331 03:52:21.213815       1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.665737467s to process 1 records
I0331 03:52:21.953199       1 tasks_processing.go:74] worker 24 stopped.
I0331 03:52:21.953601       1 recorder.go:75] Recording config/serviceaccounts with fingerprint=e03550ff5cb13d4cd718a41a9755571c983753e7ec8a789b8fb921884bbb0f64
I0331 03:52:21.953622       1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.408215968s to process 1 records
E0331 03:52:21.953693       1 periodic.go:247] "Unhandled Error" err="clusterconfig failed after 13.409s with: function \"machines\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"support_secret\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"ingress_certificates\" failed with an error, function \"config_maps\" failed with an error, function \"dvo_metrics\" failed with an error"
I0331 03:52:21.954795       1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "machines" failed with an error, function "machine_healthchecks" failed with an error, function "support_secret" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "ingress_certificates" failed with an error, function "config_maps" failed with an error, function "dvo_metrics" failed with an error
I0331 03:52:21.954809       1 periodic.go:209] Running workloads gatherer
I0331 03:52:21.954825       1 tasks_processing.go:45] number of workers: 2
I0331 03:52:21.954836       1 tasks_processing.go:69] worker 1 listening for tasks.
I0331 03:52:21.954840       1 tasks_processing.go:71] worker 1 working on workload_info task.
I0331 03:52:21.954852       1 tasks_processing.go:69] worker 0 listening for tasks.
I0331 03:52:21.954879       1 tasks_processing.go:71] worker 0 working on helmchart_info task.
I0331 03:52:21.974842       1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 18s for image data
I0331 03:52:21.983930       1 gather_workloads_info.go:387] No image sha256:ca1344cb64140188b7cae7bbc51fb751566c0b0c97d5e39b5850e628032c4a5e (10ms)
I0331 03:52:21.985606       1 tasks_processing.go:74] worker 0 stopped.
I0331 03:52:21.985621       1 gather.go:177] gatherer "workloads" function "helmchart_info" took 30.712999ms to process 0 records
I0331 03:52:21.992349       1 gather_workloads_info.go:387] No image sha256:7b31223098f08328f5ddea8e5b871dbbd5f5a61ec550e8956f66793c0c6031a9 (8ms)
I0331 03:52:22.000689       1 gather_workloads_info.go:387] No image sha256:2904a78e2eb73fd6a9bb94c105c2a056831fb4113fbb7b0607c50adc9d879c9b (8ms)
I0331 03:52:22.008878       1 gather_workloads_info.go:387] No image sha256:765f0d23b637f685f98a31bd47c131b03cf72a40761a3f9a9d6320faa3c33733 (8ms)
I0331 03:52:22.016765       1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (8ms)
I0331 03:52:22.024454       1 gather_workloads_info.go:387] No image sha256:36b9e89c3cfcf1ab9ae500486e38afb6862cba48cb0b4d84a09508ab8f3d299f (8ms)
I0331 03:52:22.037332       1 gather_workloads_info.go:387] No image sha256:a498046d64605bcccee2440aa4f04a4602baaae263cf01d977ec5208e876b1fd (13ms)
I0331 03:52:22.045470       1 gather_workloads_info.go:387] No image sha256:a0105d1eb62cf6ac9e5e2ef28d3e89bf6dc514bc594fc7090fe5a5ee18a09c87 (8ms)
I0331 03:52:22.055997       1 gather_workloads_info.go:387] No image sha256:4556896f77307821531ef91b7b7faccb82b824ea695693b2989f597f0deca038 (11ms)
I0331 03:52:22.063987       1 gather_workloads_info.go:387] No image sha256:a258c226562adb14e3a163a1940938526ee6a0928982a7667d85d9a7334ce639 (8ms)
I0331 03:52:22.085783       1 gather_workloads_info.go:387] No image sha256:695cf2f0cc07683c2a3ce1eaf3e56fe18abc6e2bac716f7d9843f5d173b9df52 (22ms)
I0331 03:52:22.183784       1 gather_workloads_info.go:387] No image sha256:56a85660a445eced5c79a595a0eccf590087c5672d50f49d4c25ad52f9a44f04 (98ms)
I0331 03:52:22.286282       1 gather_workloads_info.go:387] No image sha256:e84cb128d930bd1ab867cc89b7b7bf2b2c0e41105ab93b5381069945b3ee9c57 (102ms)
I0331 03:52:22.384618       1 gather_workloads_info.go:387] No image sha256:653c666f842c13e0baae2e89a9b1efe0e2ef56f621ffb5b32005115d2a26ab8c (98ms)
I0331 03:52:22.484549       1 gather_workloads_info.go:387] No image sha256:b3909bf664c77097f75b3768830863d642eed3815dab2bfb4415c771ca2d5007 (100ms)
I0331 03:52:22.583723       1 gather_workloads_info.go:387] No image sha256:5a95c19d82767e0235b4edb4a0536482c816904897aae1dc3eb255cb52b87a9f (99ms)
I0331 03:52:22.662718       1 configmapobserver.go:84] configmaps "insights-config" not found
I0331 03:52:22.683635       1 gather_workloads_info.go:387] No image sha256:943018739e3db1763c3184b460dbc409e058abbac76d57b9927faad317be85e4 (100ms)
I0331 03:52:22.784028       1 gather_workloads_info.go:387] No image sha256:47154813651033d59751fb655a384dbffb64dd26f10bd7f3be0c3128d0486356 (100ms)
I0331 03:52:22.867195       1 configmapobserver.go:84] configmaps "insights-config" not found
I0331 03:52:22.883693       1 gather_workloads_info.go:387] No image sha256:03cf4cd7ef1518610c6c7b3ad27d1622d82e98e3dc6e3f8e5d0fceb5c8d3786e (100ms)
I0331 03:52:22.984111       1 gather_workloads_info.go:387] No image sha256:91828234f107c068c8a4966d08370ae7b73e637651dbc6d92c18c4553402c22c (100ms)
I0331 03:52:23.084237       1 gather_workloads_info.go:387] No image sha256:7adc1eab05d6724c76ba751f6df816b08d6e70b78dee9eb94fa6fd9690542c98 (100ms)
I0331 03:52:23.183840       1 gather_workloads_info.go:387] No image sha256:0a99240166165eb5718e7516a43282fe32df9c7c5e809b31b58abe44e42ff94d (100ms)
I0331 03:52:23.284070       1 gather_workloads_info.go:387] No image sha256:2e564f336c77116053f34d4201d364d8da04e789cfffa0ea422574c95f2d6404 (100ms)
I0331 03:52:23.383094       1 gather_workloads_info.go:387] No image sha256:5f0b67cfbbc381243fb91ccc17345b56d05f4d717c667e8c644e5bf05633ba71 (99ms)
I0331 03:52:23.484079       1 gather_workloads_info.go:387] No image sha256:c940ea87e7d133d75ba0002ef00c0806825eed3db8094cdb260d1bac18127733 (101ms)
I0331 03:52:23.583724       1 gather_workloads_info.go:387] No image sha256:a56211d075aa43cbb491f669a5b2e46ee023dc95b7d51dbac28f463948c5ad61 (100ms)
I0331 03:52:23.686111       1 gather_workloads_info.go:387] No image sha256:1a2532940843248c57d52141185dd71fbc393ab28b65d48f682038632c1dbbad (102ms)
I0331 03:52:23.783961       1 gather_workloads_info.go:387] No image sha256:2e57e192c3c1240fd935dcd55c8fde5e70e78bf81d6176c96edf21fafe59f8ba (98ms)
I0331 03:52:23.884384       1 gather_workloads_info.go:387] No image sha256:521712486e2c6e3c020dad6a1cb340db8e55665b69f7c208fab9cd9e965fd588 (100ms)
I0331 03:52:23.983825       1 gather_workloads_info.go:387] No image sha256:2598489729a4b258e4ecda4a06f6875133f2a10ced5c5241f8a57a8a05418e36 (99ms)
I0331 03:52:23.983850       1 tasks_processing.go:74] worker 1 stopped.
E0331 03:52:23.983859       1 gather.go:140] gatherer "workloads" function "workload_info" failed with the error: no running pods found for the insights-runtime-extractor statefulset
I0331 03:52:23.984083       1 recorder.go:75] Recording config/workload_info with fingerprint=6fab530fb4c5d85bd2a2a76895177d9589e613f4f76da857f217d68d4b0cf768
I0331 03:52:23.984098       1 gather.go:177] gatherer "workloads" function "workload_info" took 2.029002604s to process 1 records
E0331 03:52:23.984119       1 periodic.go:247] "Unhandled Error" err="workloads failed after 2.029s with: function \"workload_info\" failed with an error"
I0331 03:52:23.985192       1 controllerstatus.go:89] name=periodic-workloads healthy=false reason=PeriodicGatherFailed message=Source workloads could not be retrieved: function "workload_info" failed with an error
I0331 03:52:23.985205       1 periodic.go:209] Running conditional gatherer
I0331 03:52:23.990864       1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules
I0331 03:52:23.997301       1 conditional_gatherer.go:107] Get "https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules": dial tcp: lookup console.redhat.com on 172.30.0.10:53: read udp 10.128.0.12:42543->172.30.0.10:53: read: connection refused
E0331 03:52:23.997546       1 conditional_gatherer.go:322] unable to update alerts cache: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory
I0331 03:52:23.997601       1 conditional_gatherer.go:384] updating version cache for conditional gatherer
I0331 03:52:24.003215       1 conditional_gatherer.go:392] cluster version is '4.20.8'
E0331 03:52:24.003228       1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing
E0331 03:52:24.003234       1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing
E0331 03:52:24.003237       1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing
E0331 03:52:24.003240       1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing
E0331 03:52:24.003243       1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing
E0331 03:52:24.003246       1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing
E0331 03:52:24.003249       1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing
E0331 03:52:24.003252       1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing
E0331 03:52:24.003255       1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing
I0331 03:52:24.003272       1 tasks_processing.go:45] number of workers: 3
I0331 03:52:24.003285       1 tasks_processing.go:69] worker 2 listening for tasks.
I0331 03:52:24.003288       1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task.
I0331 03:52:24.003293       1 tasks_processing.go:69] worker 0 listening for tasks.
I0331 03:52:24.003303       1 tasks_processing.go:71] worker 0 working on remote_configuration task.
I0331 03:52:24.003309       1 tasks_processing.go:71] worker 2 working on rapid_container_logs task.
I0331 03:52:24.003309       1 tasks_processing.go:69] worker 1 listening for tasks.
I0331 03:52:24.003317       1 tasks_processing.go:74] worker 1 stopped.
I0331 03:52:24.003372       1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=7034af97e7e41c22e4b775abdd4b9066c8ebb19da33eb7f69f39bfd2eb5f6406
I0331 03:52:24.003385       1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 712ns to process 1 records
I0331 03:52:24.003421       1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=0394430c431eec4d48bb1811a90918e95161d2282c59af26f2473613cc0959db
I0331 03:52:24.003431       1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.012µs to process 1 records
I0331 03:52:24.003437       1 tasks_processing.go:74] worker 0 stopped.
I0331 03:52:24.003548       1 tasks_processing.go:74] worker 2 stopped.
I0331 03:52:24.003558       1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 233.971µs to process 0 records
I0331 03:52:24.003576       1 controllerstatus.go:89] name=periodic-conditional healthy=false reason=NotAvailable message=Get "https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules": dial tcp: lookup console.redhat.com on 172.30.0.10:53: read udp 10.128.0.12:42543->172.30.0.10:53: read: connection refused
I0331 03:52:24.003591       1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=359de9c990c741675cec72fda96b5c3682221efdb4799f5eaa6e9805bcd3b5c1
I0331 03:52:24.025411       1 recorder.go:75] Recording insights-operator/gathers with fingerprint=9c8e34f5e2c60f25c6708d63b805c14e5a8e0da2da404308284b989cdd366893
I0331 03:52:24.025531       1 diskrecorder.go:70] Writing 100 records to /var/lib/insights-operator/insights-2026-03-31-035224.tar.gz
I0331 03:52:24.031566       1 diskrecorder.go:51] Wrote 100 records to disk in 6ms
I0331 03:52:24.031592       1 periodic.go:278] Gathering cluster info every 2h0m0s
I0331 03:52:24.031607       1 periodic.go:279] Configuration is 
	dataReporting:
		interval: 2h0m0s,
		uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload,
		storagePath: /var/lib/insights-operator,
		downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, 
		conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules,
		obfuscation: []
	sca:
		disabled: false,
		endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates,
		interval: 8h0m0s
	alerting:
		disabled: false
	clusterTransfer:
		endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/,
		interval: 12h0m0s
	proxy:
		httpProxy: ,
		httpsProxy: ,
		noProxy: 
I0331 03:52:33.239428       1 configmapobserver.go:84] configmaps "insights-config" not found
I0331 03:53:13.047817       1 observer_polling.go:111] Observed file "/var/run/configmaps/service-ca-bundle/service-ca.crt" has been created (hash="bb6691c8f5ac4aa1cf2e21cedcbe78c58543e6479f60b13d6458746689800a4e")
W0331 03:53:13.047853       1 builder.go:160] Restart triggered because of file /var/run/configmaps/service-ca-bundle/service-ca.crt was created
I0331 03:53:13.047917       1 observer_polling.go:111] Observed file "/var/run/secrets/serving-cert/tls.crt" has been created (hash="7c48e2674177eefd39d1b2a05549c35b6649302db2ffaf74d9b73e11819dffeb")
I0331 03:53:13.047960       1 observer_polling.go:111] Observed file "/var/run/secrets/serving-cert/tls.key" has been created (hash="a334ddcee217918e9756963916174318e231b830a4a95c0f44ec820347cf0873")
I0331 03:53:13.047973       1 genericapiserver.go:548] "[graceful-termination] shutdown event" name="ShutdownInitiated"
I0331 03:53:13.047993       1 genericapiserver.go:693] "[graceful-termination] pre-shutdown hooks completed" name="PreShutdownHooksStopped"
I0331 03:53:13.048027       1 configmap_cafile_content.go:226] "Shutting down controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file"
I0331 03:53:13.048045       1 genericapiserver.go:651] "[graceful-termination] not going to wait for active watch request(s) to drain"
I0331 03:53:13.048050       1 base_controller.go:181] Shutting down ConfigController ...
I0331 03:53:13.048048       1 requestheader_controller.go:194] Shutting down RequestHeaderAuthRequestController
I0331 03:53:13.048052       1 object_count_tracker.go:151] "StorageObjectCountTracker pruner is exiting"
I0331 03:53:13.048058       1 configmap_cafile_content.go:226] "Shutting down controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file"
I0331 03:53:13.048029       1 simple_featuregate_reader.go:177] Shutting down feature-gate-detector
I0331 03:53:13.048074       1 base_controller.go:123] Shutting down worker of ConfigController controller ...
I0331 03:53:13.048101       1 dynamic_serving_content.go:149] "Shutting down controller" name="serving-cert::/tmp/serving-cert-18288637/tls.crt::/tmp/serving-cert-18288637/tls.key"
E0331 03:53:13.048103       1 controller.go:299] Unable to write cluster operator status: client rate limiter Wait returned an error: context canceled
I0331 03:53:13.048103       1 base_controller.go:113] All ConfigController workers have been terminated
I0331 03:53:13.048109       1 secure_serving.go:258] Stopped listening on [::]:8443
I0331 03:53:13.048119       1 base_controller.go:181] Shutting down LoggingSyncer ...
I0331 03:53:13.048123       1 periodic.go:170] Shutting down
I0331 03:53:13.048127       1 genericapiserver.go:600] "[graceful-termination] shutdown event" name="HTTPServerStoppedListening"