I0420 09:02:21.745684 1 cmd.go:253] Using service-serving-cert provided certificates I0420 09:02:21.746329 1 observer_polling.go:159] Starting file observer I0420 09:02:22.097416 1 operator.go:60] Starting insights-operator v0.0.0-master+$Format:%H$ I0420 09:02:22.097609 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0420 09:02:22.098205 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0420 09:02:22.098314 1 secure_serving.go:57] Forcing use of http/1.1 only W0420 09:02:22.098332 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0420 09:02:22.098336 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0420 09:02:22.098341 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0420 09:02:22.098344 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0420 09:02:22.098346 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0420 09:02:22.098349 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0420 09:02:22.101721 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"9441fac0-6427-4deb-bd2e-1e59adc94c91", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}} I0420 09:02:22.101705 1 operator.go:125] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GatewayAPI GatewayAPIController HighlyAvailableArbiter ImageVolume IngressControllerLBSubnetsAWS KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController SigstoreImageVerification StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPClusterHostedDNSInstall GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall ImageModeStatusReporting ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesAzure ManagedBootImagesvSphere MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutatingAdmissionPolicy NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterOperations NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PreconfiguredUDNAddresses SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeAttributesClass VolumeGroupSnapshot] I0420 09:02:22.104932 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0420 09:02:22.104942 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0420 09:02:22.104952 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0420 09:02:22.104957 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0420 09:02:22.104963 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0420 09:02:22.104957 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0420 09:02:22.105247 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0420 09:02:22.105455 1 secure_serving.go:213] Serving securely on [::]:8443 I0420 09:02:22.105480 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0420 09:02:22.106725 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0420 09:02:22.106745 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0420 09:02:22.106842 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0420 09:02:22.112401 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 09:02:22.112417 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0420 09:02:22.116651 1 secretconfigobserver.go:119] support secret does not exist I0420 09:02:22.121627 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 09:02:22.125744 1 secretconfigobserver.go:119] support secret does not exist I0420 09:02:22.127770 1 recorder.go:161] Pruning old reports every 5h15m47s, max age is 288h0m0s I0420 09:02:22.132646 1 periodic.go:209] Running clusterconfig gatherer I0420 09:02:22.132659 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0420 09:02:22.132667 1 insightsreport.go:296] Starting report retriever I0420 09:02:22.132671 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0420 09:02:22.132646 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0420 09:02:22.132710 1 tasks_processing.go:45] number of workers: 64 I0420 09:02:22.132714 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0420 09:02:22.132741 1 tasks_processing.go:69] worker 3 listening for tasks. I0420 09:02:22.132756 1 tasks_processing.go:69] worker 0 listening for tasks. I0420 09:02:22.132755 1 tasks_processing.go:69] worker 6 listening for tasks. I0420 09:02:22.132762 1 tasks_processing.go:69] worker 5 listening for tasks. I0420 09:02:22.132770 1 tasks_processing.go:69] worker 7 listening for tasks. I0420 09:02:22.132773 1 tasks_processing.go:69] worker 4 listening for tasks. I0420 09:02:22.132776 1 tasks_processing.go:69] worker 1 listening for tasks. I0420 09:02:22.132778 1 tasks_processing.go:69] worker 2 listening for tasks. I0420 09:02:22.132789 1 tasks_processing.go:69] worker 9 listening for tasks. I0420 09:02:22.132788 1 tasks_processing.go:69] worker 8 listening for tasks. I0420 09:02:22.132848 1 tasks_processing.go:69] worker 18 listening for tasks. I0420 09:02:22.132858 1 tasks_processing.go:69] worker 10 listening for tasks. I0420 09:02:22.132859 1 tasks_processing.go:69] worker 30 listening for tasks. I0420 09:02:22.132867 1 tasks_processing.go:69] worker 11 listening for tasks. I0420 09:02:22.132872 1 tasks_processing.go:69] worker 19 listening for tasks. I0420 09:02:22.132875 1 tasks_processing.go:69] worker 24 listening for tasks. I0420 09:02:22.132876 1 tasks_processing.go:69] worker 12 listening for tasks. I0420 09:02:22.132879 1 tasks_processing.go:69] worker 25 listening for tasks. I0420 09:02:22.132878 1 tasks_processing.go:69] worker 36 listening for tasks. I0420 09:02:22.132887 1 tasks_processing.go:69] worker 26 listening for tasks. I0420 09:02:22.132890 1 tasks_processing.go:69] worker 33 listening for tasks. I0420 09:02:22.132894 1 tasks_processing.go:69] worker 27 listening for tasks. I0420 09:02:22.132895 1 tasks_processing.go:69] worker 34 listening for tasks. I0420 09:02:22.132901 1 tasks_processing.go:69] worker 31 listening for tasks. I0420 09:02:22.132897 1 tasks_processing.go:69] worker 20 listening for tasks. I0420 09:02:22.132907 1 tasks_processing.go:69] worker 21 listening for tasks. I0420 09:02:22.132917 1 tasks_processing.go:69] worker 22 listening for tasks. I0420 09:02:22.132919 1 tasks_processing.go:69] worker 32 listening for tasks. I0420 09:02:22.132922 1 tasks_processing.go:69] worker 28 listening for tasks. I0420 09:02:22.132925 1 tasks_processing.go:69] worker 23 listening for tasks. I0420 09:02:22.132926 1 tasks_processing.go:69] worker 40 listening for tasks. I0420 09:02:22.132930 1 tasks_processing.go:69] worker 37 listening for tasks. I0420 09:02:22.132932 1 tasks_processing.go:69] worker 35 listening for tasks. I0420 09:02:22.132937 1 tasks_processing.go:69] worker 41 listening for tasks. I0420 09:02:22.132939 1 tasks_processing.go:69] worker 14 listening for tasks. I0420 09:02:22.132943 1 tasks_processing.go:69] worker 38 listening for tasks. I0420 09:02:22.132932 1 tasks_processing.go:69] worker 29 listening for tasks. I0420 09:02:22.132943 1 tasks_processing.go:69] worker 39 listening for tasks. I0420 09:02:22.132932 1 tasks_processing.go:69] worker 16 listening for tasks. I0420 09:02:22.132962 1 tasks_processing.go:69] worker 13 listening for tasks. I0420 09:02:22.132963 1 tasks_processing.go:69] worker 15 listening for tasks. I0420 09:02:22.132943 1 tasks_processing.go:69] worker 17 listening for tasks. I0420 09:02:22.133002 1 tasks_processing.go:71] worker 17 working on authentication task. I0420 09:02:22.133011 1 tasks_processing.go:71] worker 8 working on cost_management_metrics_configs task. I0420 09:02:22.133016 1 tasks_processing.go:71] worker 18 working on service_accounts task. I0420 09:02:22.133021 1 tasks_processing.go:71] worker 14 working on ingress task. I0420 09:02:22.133022 1 tasks_processing.go:71] worker 13 working on feature_gates task. I0420 09:02:22.133039 1 tasks_processing.go:71] worker 23 working on install_plans task. I0420 09:02:22.133050 1 tasks_processing.go:71] worker 30 working on jaegers task. I0420 09:02:22.133053 1 tasks_processing.go:71] worker 16 working on machine_configs task. I0420 09:02:22.133016 1 tasks_processing.go:71] worker 39 working on openshift_logging task. I0420 09:02:22.133071 1 tasks_processing.go:71] worker 19 working on dvo_metrics task. I0420 09:02:22.133011 1 tasks_processing.go:71] worker 21 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0420 09:02:22.133141 1 tasks_processing.go:71] worker 27 working on support_secret task. I0420 09:02:22.133148 1 tasks_processing.go:71] worker 38 working on machine_sets task. I0420 09:02:22.133989 1 tasks_processing.go:69] worker 42 listening for tasks. I0420 09:02:22.134252 1 tasks_processing.go:71] worker 42 working on nodenetworkstates task. I0420 09:02:22.134266 1 tasks_processing.go:69] worker 45 listening for tasks. I0420 09:02:22.134320 1 tasks_processing.go:69] worker 62 listening for tasks. I0420 09:02:22.134339 1 tasks_processing.go:69] worker 46 listening for tasks. I0420 09:02:22.134390 1 tasks_processing.go:69] worker 47 listening for tasks. I0420 09:02:22.134417 1 tasks_processing.go:71] worker 7 working on mutating_webhook_configurations task. I0420 09:02:22.134423 1 tasks_processing.go:71] worker 5 working on olm_operators task. I0420 09:02:22.134006 1 tasks_processing.go:71] worker 40 working on operators task. I0420 09:02:22.134253 1 tasks_processing.go:69] worker 44 listening for tasks. I0420 09:02:22.134036 1 tasks_processing.go:69] worker 58 listening for tasks. I0420 09:02:22.133982 1 tasks_processing.go:71] worker 11 working on metrics task. I0420 09:02:22.134711 1 tasks_processing.go:71] worker 58 working on sap_config task. I0420 09:02:22.134740 1 tasks_processing.go:71] worker 47 working on lokistack task. I0420 09:02:22.134748 1 tasks_processing.go:71] worker 45 working on qemu_kubevirt_launcher_logs task. I0420 09:02:22.134787 1 tasks_processing.go:71] worker 31 working on cluster_apiserver task. I0420 09:02:22.134788 1 tasks_processing.go:69] worker 51 listening for tasks. I0420 09:02:22.134895 1 tasks_processing.go:71] worker 51 working on openstack_version task. I0420 09:02:22.134924 1 tasks_processing.go:71] worker 4 working on openstack_dataplanedeployments task. I0420 09:02:22.134930 1 tasks_processing.go:69] worker 54 listening for tasks. I0420 09:02:22.134950 1 tasks_processing.go:69] worker 52 listening for tasks. I0420 09:02:22.134959 1 tasks_processing.go:69] worker 53 listening for tasks. I0420 09:02:22.134968 1 tasks_processing.go:69] worker 55 listening for tasks. I0420 09:02:22.135011 1 tasks_processing.go:71] worker 2 working on schedulers task. I0420 09:02:22.135070 1 tasks_processing.go:71] worker 24 working on container_images task. I0420 09:02:22.134741 1 tasks_processing.go:71] worker 46 working on config_maps task. I0420 09:02:22.135199 1 tasks_processing.go:71] worker 53 working on proxies task. I0420 09:02:22.135271 1 tasks_processing.go:69] worker 56 listening for tasks. I0420 09:02:22.135284 1 tasks_processing.go:71] worker 56 working on ingress_certificates task. I0420 09:02:22.133989 1 tasks_processing.go:71] worker 36 working on pdbs task. I0420 09:02:22.134097 1 tasks_processing.go:71] worker 26 working on crds task. I0420 09:02:22.134455 1 tasks_processing.go:69] worker 48 listening for tasks. I0420 09:02:22.134134 1 tasks_processing.go:69] worker 57 listening for tasks. I0420 09:02:22.134163 1 tasks_processing.go:71] worker 37 working on silenced_alerts task. I0420 09:02:22.135075 1 tasks_processing.go:71] worker 9 working on networks task. I0420 09:02:22.135341 1 tasks_processing.go:71] worker 29 working on container_runtime_configs task. I0420 09:02:22.133044 1 tasks_processing.go:71] worker 25 working on storage_classes task. I0420 09:02:22.134243 1 tasks_processing.go:71] worker 35 working on active_alerts task. I0420 09:02:22.134239 1 tasks_processing.go:69] worker 43 listening for tasks. I0420 09:02:22.134256 1 tasks_processing.go:69] worker 61 listening for tasks. I0420 09:02:22.134502 1 tasks_processing.go:69] worker 63 listening for tasks. I0420 09:02:22.134542 1 tasks_processing.go:69] worker 49 listening for tasks. I0420 09:02:22.134555 1 tasks_processing.go:71] worker 3 working on image_registries task. I0420 09:02:22.134229 1 tasks_processing.go:69] worker 59 listening for tasks. I0420 09:02:22.134603 1 tasks_processing.go:69] worker 50 listening for tasks. I0420 09:02:22.134043 1 tasks_processing.go:71] worker 15 working on monitoring_persistent_volumes task. I0420 09:02:22.134741 1 tasks_processing.go:71] worker 6 working on infrastructures task. I0420 09:02:22.134854 1 tasks_processing.go:71] worker 1 working on tsdb_status task. I0420 09:02:22.134244 1 tasks_processing.go:69] worker 60 listening for tasks. I0420 09:02:22.133047 1 tasks_processing.go:71] worker 10 working on machines task. I0420 09:02:22.134858 1 tasks_processing.go:71] worker 62 working on aggregated_monitoring_cr_names task. I0420 09:02:22.135586 1 tasks_processing.go:71] worker 60 working on machine_healthchecks task. I0420 09:02:22.135651 1 tasks_processing.go:71] worker 48 working on version task. I0420 09:02:22.135084 1 tasks_processing.go:71] worker 12 working on nodes task. I0420 09:02:22.135713 1 tasks_processing.go:71] worker 43 working on storage_cluster task. I0420 09:02:22.135658 1 tasks_processing.go:71] worker 63 working on openshift_machine_api_events task. I0420 09:02:22.135136 1 tasks_processing.go:71] worker 55 working on overlapping_namespace_uids task. I0420 09:02:22.135764 1 tasks_processing.go:71] worker 49 working on oauths task. I0420 09:02:22.135179 1 tasks_processing.go:71] worker 34 working on clusterroles task. I0420 09:02:22.135167 1 tasks_processing.go:71] worker 20 working on ceph_cluster task. I0420 09:02:22.135186 1 tasks_processing.go:71] worker 54 working on pod_network_connectivity_checks task. I0420 09:02:22.135196 1 tasks_processing.go:71] worker 52 working on openstack_controlplanes task. I0420 09:02:22.134120 1 tasks_processing.go:71] worker 33 working on machine_autoscalers task. I0420 09:02:22.134127 1 tasks_processing.go:71] worker 41 working on image_pruners task. I0420 09:02:22.134595 1 tasks_processing.go:71] worker 0 working on sap_pods task. I0420 09:02:22.135090 1 tasks_processing.go:71] worker 44 working on image task. I0420 09:02:22.135664 1 tasks_processing.go:71] worker 61 working on nodenetworkconfigurationpolicies task. I0420 09:02:22.135671 1 tasks_processing.go:71] worker 50 working on validating_webhook_configurations task. I0420 09:02:22.135670 1 tasks_processing.go:71] worker 59 working on openstack_dataplanenodesets task. I0420 09:02:22.135709 1 tasks_processing.go:71] worker 57 working on sap_datahubs task. I0420 09:02:22.136520 1 tasks_processing.go:71] worker 28 working on certificate_signing_requests task. I0420 09:02:22.136628 1 tasks_processing.go:71] worker 22 working on machine_config_pools task. I0420 09:02:22.137571 1 tasks_processing.go:71] worker 32 working on operators_pods_and_events task. I0420 09:02:22.139012 1 tasks_processing.go:71] worker 30 working on node_logs task. I0420 09:02:22.139025 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 5.950236ms to process 0 records I0420 09:02:22.139136 1 tasks_processing.go:74] worker 38 stopped. I0420 09:02:22.139153 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 5.129185ms to process 0 records I0420 09:02:22.139360 1 controller.go:128] Initializing last reported time to 2026-04-20T09:02:15Z I0420 09:02:22.139374 1 controller.go:326] The initial operator extension status is healthy I0420 09:02:22.139381 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0420 09:02:22.139387 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0420 09:02:22.139390 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0420 09:02:22.139403 1 controller.go:489] The operator is still being initialized I0420 09:02:22.139410 1 controller.go:512] The operator is healthy E0420 09:02:22.139472 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.181.39:9091: connect: connection refused I0420 09:02:22.139494 1 tasks_processing.go:74] worker 1 stopped. E0420 09:02:22.139484 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.181.39:9091: connect: connection refused E0420 09:02:22.139503 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.181.39:9091: connect: connection refused I0420 09:02:22.139513 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 3.99654ms to process 0 records E0420 09:02:22.139532 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.181.39:9091: connect: connection refused I0420 09:02:22.139553 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 4.789381ms to process 0 records I0420 09:02:22.139565 1 tasks_processing.go:74] worker 11 stopped. I0420 09:02:22.144761 1 tasks_processing.go:74] worker 47 stopped. I0420 09:02:22.144778 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 10.00684ms to process 0 records I0420 09:02:22.144788 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 9.846243ms to process 0 records I0420 09:02:22.144796 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 9.877502ms to process 0 records I0420 09:02:22.144802 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 11.724847ms to process 0 records I0420 09:02:22.144801 1 tasks_processing.go:74] worker 4 stopped. I0420 09:02:22.144808 1 tasks_processing.go:74] worker 39 stopped. I0420 09:02:22.144812 1 tasks_processing.go:74] worker 51 stopped. I0420 09:02:22.144860 1 tasks_processing.go:74] worker 54 stopped. E0420 09:02:22.144872 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0420 09:02:22.144879 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 8.992177ms to process 0 records I0420 09:02:22.144992 1 tasks_processing.go:74] worker 10 stopped. E0420 09:02:22.145003 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0420 09:02:22.145011 1 gather.go:177] gatherer "clusterconfig" function "machines" took 9.4717ms to process 0 records I0420 09:02:22.145023 1 tasks_processing.go:74] worker 2 stopped. I0420 09:02:22.145133 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=59ae95616c7c78aadad02adf0a49836ae6fa71969b76926309e42dd3fb935e89 I0420 09:02:22.145148 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 9.993618ms to process 1 records I0420 09:02:22.145236 1 tasks_processing.go:74] worker 13 stopped. I0420 09:02:22.145247 1 recorder.go:75] Recording config/featuregate with fingerprint=8a4fb80e2fe7b878f58843461b94ff46f5f428b9def68d2f536b8d98210b4b14 I0420 09:02:22.145252 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 12.041104ms to process 1 records I0420 09:02:22.145294 1 recorder.go:75] Recording config/proxy with fingerprint=ec14163cbad12c83a1fb426a1971f5b9fe2f8eaf7c27476549aab4fa19295131 I0420 09:02:22.145302 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 9.846881ms to process 1 records I0420 09:02:22.145327 1 tasks_processing.go:74] worker 53 stopped. I0420 09:02:22.145393 1 tasks_processing.go:74] worker 41 stopped. I0420 09:02:22.145493 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=9b2050bf0879367ef8be7ac759e55cfdd1e76873b74c256a2c27b7e57c428e5e I0420 09:02:22.145501 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 9.254538ms to process 1 records I0420 09:02:22.145576 1 tasks_processing.go:74] worker 17 stopped. I0420 09:02:22.145625 1 recorder.go:75] Recording config/authentication with fingerprint=6f9b7876a57df351fd290281da18a28a498d1b22eaa5fabfbd91e371c144f2bf I0420 09:02:22.145632 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 12.315821ms to process 1 records I0420 09:02:22.152849 1 tasks_processing.go:74] worker 25 stopped. I0420 09:02:22.152942 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=954825a29c3cfe1ac5d4b0623864a47d0b7ce53998abc69db873fd8fb1021bc4 I0420 09:02:22.152962 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=f43ed03f2bb1c386b917accaf503b4c6dd91077877659a0075f61cb83931ee93 I0420 09:02:22.152969 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 17.432981ms to process 2 records I0420 09:02:22.153089 1 tasks_processing.go:74] worker 7 stopped. I0420 09:02:22.153588 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=2c6b9461f5250636007be2091173d48bca22a92672341f5deb3ca095b38582d9 I0420 09:02:22.153639 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=34d40cad4661318b7ff4405761768cf8db1524b4f310e60c03eddfbdbdccc496 I0420 09:02:22.153730 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=3f99eaf40e041715b374bb5c1e400ad14851ae054cea6a121674fb3d6de4765f I0420 09:02:22.153744 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 18.452895ms to process 3 records I0420 09:02:22.158855 1 tasks_processing.go:74] worker 20 stopped. I0420 09:02:22.158868 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 23.040428ms to process 0 records I0420 09:02:22.161961 1 tasks_processing.go:74] worker 33 stopped. I0420 09:02:22.161973 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 26.013586ms to process 0 records I0420 09:02:22.163152 1 tasks_processing.go:74] worker 57 stopped. I0420 09:02:22.163165 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 26.911413ms to process 0 records I0420 09:02:22.163363 1 tasks_processing.go:74] worker 29 stopped. I0420 09:02:22.163379 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 27.962863ms to process 0 records I0420 09:02:22.163392 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 30.347334ms to process 0 records I0420 09:02:22.163398 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 29.101981ms to process 0 records I0420 09:02:22.163404 1 tasks_processing.go:74] worker 42 stopped. I0420 09:02:22.163409 1 tasks_processing.go:74] worker 8 stopped. I0420 09:02:22.163451 1 tasks_processing.go:74] worker 6 stopped. I0420 09:02:22.163958 1 recorder.go:75] Recording config/infrastructure with fingerprint=42a78e2faf401b484aa54addae5696e9bc6ac79ae7fb7602f7935819dc527f9f I0420 09:02:22.163972 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 27.963363ms to process 1 records I0420 09:02:22.164054 1 tasks_processing.go:74] worker 49 stopped. I0420 09:02:22.164143 1 recorder.go:75] Recording config/oauth with fingerprint=ec9f4a06a2087c1d24e83e054077d021764e14bdf92122411ee7d278009343bd I0420 09:02:22.164152 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 27.681402ms to process 1 records I0420 09:02:22.164161 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 27.747346ms to process 0 records I0420 09:02:22.164164 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 28.02627ms to process 0 records I0420 09:02:22.164171 1 tasks_processing.go:74] worker 43 stopped. I0420 09:02:22.164183 1 tasks_processing.go:74] worker 15 stopped. I0420 09:02:22.164238 1 tasks_processing.go:74] worker 14 stopped. I0420 09:02:22.164260 1 recorder.go:75] Recording config/ingress with fingerprint=3625c537d2f2d358baebb96127c5a20120387a7edbf468728f38825c59ec1b8a I0420 09:02:22.164269 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 30.564485ms to process 1 records I0420 09:02:22.164279 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 27.866276ms to process 0 records I0420 09:02:22.164292 1 tasks_processing.go:74] worker 63 stopped. I0420 09:02:22.164352 1 tasks_processing.go:74] worker 9 stopped. I0420 09:02:22.164382 1 recorder.go:75] Recording config/network with fingerprint=d3293cc000e4298fe5bdfd1cb65e9bba2642716cf0c051acb219cf64fab20fa7 I0420 09:02:22.164390 1 gather.go:177] gatherer "clusterconfig" function "networks" took 28.263224ms to process 1 records I0420 09:02:22.164473 1 tasks_processing.go:74] worker 31 stopped. I0420 09:02:22.164497 1 recorder.go:75] Recording config/apiserver with fingerprint=cfdfd72a395d4fadce8f5b719de5d733ef60d98fa5a1968572f475d98843d8bf I0420 09:02:22.164508 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 28.967249ms to process 1 records I0420 09:02:22.165665 1 tasks_processing.go:74] worker 52 stopped. I0420 09:02:22.165676 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 29.788755ms to process 0 records I0420 09:02:22.165785 1 tasks_processing.go:74] worker 36 stopped. I0420 09:02:22.165896 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=1c75107fbeda62d9730cab5e0bec74d91d48372d0b76cd733b957d617d452ee7 I0420 09:02:22.165922 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=f8f81bc0b7128ef01bd868dd20ea41915103084b2b1413753585ac566f51890e I0420 09:02:22.165944 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=6a5a528f638f8b21af06c6f281e69ece3491d84072f54c74cf95ece9de134592 I0420 09:02:22.165968 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=b96e57f9a37f2bd9e4599eb72ca4ee5596418a2bee74283f277395ff47b20439 I0420 09:02:22.165985 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=11038099defd1d268029c5b7acea668d452665fe333f3ace7430bd33205112c8 I0420 09:02:22.166004 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=d54521723613bd7ae609a4ef75d78fda2c624d62796af4ea88d19b2cbf57f14f I0420 09:02:22.166029 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=d5a05d6b2803a43b5dcada82073f555ac85955b25315918ac215ae9f854d273a I0420 09:02:22.166049 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=a69c93ba5ecabc773bbd83e2728a5d34c7851fa8ff6923e1aa13c32b5ec39780 I0420 09:02:22.166064 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=b94d9953a49c623d04ccf5766a91cf847d245de49ba57fb5b5dd9d9babd1d74a I0420 09:02:22.166087 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=5d314008332ad33645c51fa00084c4c04f4eeec3bd82bda89b3a7e071fd5a819 I0420 09:02:22.166128 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=42598d69e0fa1a092eaf66007f7c0747df028e2b7043ab6b668a1b6c221c04ab I0420 09:02:22.166138 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 30.484093ms to process 11 records I0420 09:02:22.166154 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 31.159218ms to process 0 records E0420 09:02:22.166162 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0420 09:02:22.166166 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 32.752875ms to process 0 records I0420 09:02:22.166171 1 tasks_processing.go:74] worker 27 stopped. I0420 09:02:22.166175 1 tasks_processing.go:74] worker 58 stopped. I0420 09:02:22.166216 1 tasks_processing.go:74] worker 61 stopped. I0420 09:02:22.166230 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 30.121899ms to process 0 records I0420 09:02:22.166248 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 30.042283ms to process 0 records I0420 09:02:22.166257 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 30.252041ms to process 0 records I0420 09:02:22.166264 1 tasks_processing.go:74] worker 0 stopped. I0420 09:02:22.166271 1 tasks_processing.go:74] worker 59 stopped. I0420 09:02:22.166401 1 tasks_processing.go:74] worker 60 stopped. E0420 09:02:22.166414 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0420 09:02:22.166422 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 30.750846ms to process 0 records I0420 09:02:22.166490 1 tasks_processing.go:74] worker 44 stopped. I0420 09:02:22.166580 1 recorder.go:75] Recording config/image with fingerprint=6c3e5e99050576460bbb8a47150f1bd3795d6a9eb091e07e6bc61b95b114eaf8 I0420 09:02:22.166589 1 gather.go:177] gatherer "clusterconfig" function "image" took 30.466169ms to process 1 records I0420 09:02:22.167633 1 tasks_processing.go:74] worker 30 stopped. I0420 09:02:22.167649 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 28.607542ms to process 0 records I0420 09:02:22.169619 1 tasks_processing.go:74] worker 3 stopped. W0420 09:02:22.171096 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0420 09:02:22.171369 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=baafc32d8c56619528600d961bc4b6573841600716037065d728ef795472ec06 I0420 09:02:22.171386 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 34.172238ms to process 1 records I0420 09:02:22.171473 1 tasks_processing.go:74] worker 12 stopped. I0420 09:02:22.171643 1 recorder.go:75] Recording config/node/ip-10-0-0-82.ec2.internal with fingerprint=8b0375939eeabd9bc4cfd489e17a03274b676503daf8d21145247adc2eda2d0b I0420 09:02:22.171714 1 recorder.go:75] Recording config/node/ip-10-0-1-132.ec2.internal with fingerprint=1fe69529f1378a46e0561044ac8071177d9633acd692860d65ab04130ab55bd3 I0420 09:02:22.171763 1 recorder.go:75] Recording config/node/ip-10-0-2-147.ec2.internal with fingerprint=6c3bded32e28413ec4471255642df9c26fb053ae9db5466d93d419abb6a8da40 I0420 09:02:22.171770 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 35.411541ms to process 3 records I0420 09:02:22.171837 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=cce71268c4002f1f8f3a36216bbbafa0aee107ac7cdc0f6f6effc512098d0bc7 I0420 09:02:22.171854 1 tasks_processing.go:74] worker 50 stopped. I0420 09:02:22.171883 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=8eb6251abd76f1909d1612ff99925927780160b7a78a8c10974dee5f9d32a4e8 I0420 09:02:22.171956 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=1b432ab334bc2cf54cc6f2a1cd8285c54a0d50e98436eaace7ed78b7b49342db I0420 09:02:22.171983 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=f34c67316b84fb8820dd8f571727199ce5d24403fea0de9b9f4621bebe3e8f8b I0420 09:02:22.172005 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=fac0e887bbf050a3f929b39c21c09cb3129c3004ed81545a27be64e1efe0deca I0420 09:02:22.172029 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=82da5aef9f9ac64c2644a84c588f5d59fc72950b40cb2d3075a3e30e45997c97 I0420 09:02:22.172052 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=0f485fee288a5d7d7a3b82e834c1717350cfb9de50ff4c35d538700e642f94b1 I0420 09:02:22.172075 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=903e78884b3efd36ade389d8657d6953a16a27a6e38dfa55b0e06a481ed2f617 I0420 09:02:22.172130 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=ecdbcf7c7e0ff6afb470333dfccf7ba85be09a3241b268242721f16d5fbaada0 I0420 09:02:22.172175 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=4dab3607310cd04ac0414c50000f056cf3df6c9b8416611bdab1d08dc70d2aaa I0420 09:02:22.172202 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=6ec7922d763383ede72cad60854cf7e9fab39bb509c7ce48eb1598b5c09be695 I0420 09:02:22.172224 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=6da6c381e847182cd2e4d8dd9e3f69ebd1df7d59551e9d4b615ac0046dac4fd7 I0420 09:02:22.172246 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=cc3cd286331b58a59c53b058bddb32587fccb9d923c374dede86aaf0ccf9751f I0420 09:02:22.172253 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 35.214221ms to process 13 records I0420 09:02:22.172297 1 recorder.go:75] Recording config/olm_operators with fingerprint=ed4aaf62a4f449efeadf828f2c5799c2373aaacb1131af4f77c6496e577c055b I0420 09:02:22.172304 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 37.424828ms to process 1 records I0420 09:02:22.172342 1 tasks_processing.go:74] worker 5 stopped. I0420 09:02:22.172412 1 tasks_processing.go:74] worker 26 stopped. I0420 09:02:22.172851 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=1da4be25050def5a9300614b12c3a61cfee2b4f8b3db97f19615cf5fad9cf74b I0420 09:02:22.173096 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=908c36ee9dc3d0291ca1ac9e99044f7dc634ea6156d22c6c288e8c7be1ba03e0 I0420 09:02:22.173123 1 gather.go:177] gatherer "clusterconfig" function "crds" took 36.733927ms to process 2 records I0420 09:02:22.173132 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 36.082126ms to process 0 records I0420 09:02:22.173138 1 tasks_processing.go:74] worker 28 stopped. I0420 09:02:22.176363 1 tasks_processing.go:74] worker 34 stopped. I0420 09:02:22.176676 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=12831dfdbf43557b2376392a4aa63eafe36cfc687016a92a88f93ae7216ed547 I0420 09:02:22.176713 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0420 09:02:22.176742 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s I0420 09:02:22.176847 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=9cf855c7444fa20781c6e86898cf99ad4574f5bbcb081a72b19d3e19b5bfb3cd I0420 09:02:22.176868 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 40.557884ms to process 2 records W0420 09:02:22.176971 1 operator.go:288] started I0420 09:02:22.177002 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0420 09:02:22.184270 1 tasks_processing.go:74] worker 16 stopped. I0420 09:02:22.184330 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0420 09:02:22.184350 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 51.204184ms to process 1 records I0420 09:02:22.187491 1 tasks_processing.go:74] worker 32 stopped. I0420 09:02:22.187506 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 49.899248ms to process 0 records I0420 09:02:22.189585 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0420 09:02:22.189598 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0420 09:02:22.189601 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0420 09:02:22.189606 1 controller.go:212] Source scaController *sca.Controller is not ready I0420 09:02:22.189610 1 controller.go:212] Source clusterTransferController *clustertransfer.Controller is not ready I0420 09:02:22.189630 1 controller.go:489] The operator is still being initialized I0420 09:02:22.189638 1 controller.go:512] The operator is healthy I0420 09:02:22.190191 1 tasks_processing.go:74] worker 62 stopped. I0420 09:02:22.190208 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 54.659548ms to process 0 records I0420 09:02:22.191595 1 tasks_processing.go:74] worker 24 stopped. I0420 09:02:22.193408 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-575cd97545-qs68s with fingerprint=47957d4efb393c8c3e96470abb84ad11d0d3d60e7d9ac8308818c23af2a48da3 I0420 09:02:22.193523 1 recorder.go:75] Recording config/running_containers with fingerprint=4857dda1879cba08423a23867b701fed56a7136ad9a645ae9b125bf28f5d7a27 I0420 09:02:22.193536 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 56.431309ms to process 2 records I0420 09:02:22.200448 1 tasks_processing.go:74] worker 46 stopped. E0420 09:02:22.200462 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0420 09:02:22.200468 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0420 09:02:22.200471 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0420 09:02:22.200480 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=26b6661162b099a0f5a279859b4f46c867929a79d9a4a41fde4be4e6fe138018 I0420 09:02:22.200503 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0420 09:02:22.200507 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0420 09:02:22.200511 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=c93090eb0d2a4736885abeb79c91680cfd01fda46464f83456b085d4dc8239f0 I0420 09:02:22.200519 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0420 09:02:22.200565 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0420 09:02:22.200572 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0420 09:02:22.200577 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 65.236868ms to process 7 records I0420 09:02:22.205187 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0420 09:02:22.205209 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0420 09:02:22.205219 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0420 09:02:22.206979 1 base_controller.go:82] Caches are synced for ConfigController I0420 09:02:22.206989 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0420 09:02:22.208245 1 tasks_processing.go:74] worker 56 stopped. E0420 09:02:22.208262 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0420 09:02:22.208274 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2ppphakho7bv59m4jq6j95q87iepucal-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2ppphakho7bv59m4jq6j95q87iepucal-primary-cert-bundle-secret" not found I0420 09:02:22.208332 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=9a97c80c96e1c67636fcac4725ab7a4351c4e4b37cfbd43a97bc917d0f68b552 I0420 09:02:22.208348 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 72.948683ms to process 1 records I0420 09:02:22.214788 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 09:02:22.245554 1 tasks_processing.go:74] worker 22 stopped. I0420 09:02:22.245572 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 108.872492ms to process 0 records I0420 09:02:22.246382 1 tasks_processing.go:74] worker 48 stopped. I0420 09:02:22.246624 1 recorder.go:75] Recording config/version with fingerprint=3e53cfd7a94efe8b87f7486802ef99bb7bc5331028222f3d3a6607ebc67f3707 I0420 09:02:22.246638 1 recorder.go:75] Recording config/id with fingerprint=815962f5b7a0e9abcc370ba25248278a942b4df96152e327c3df15eb99cee262 I0420 09:02:22.246647 1 gather.go:177] gatherer "clusterconfig" function "version" took 110.6954ms to process 2 records I0420 09:02:22.246743 1 gather_logs.go:145] no pods in namespace were found I0420 09:02:22.246757 1 tasks_processing.go:74] worker 45 stopped. I0420 09:02:22.246770 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 111.977368ms to process 0 records I0420 09:02:22.247620 1 tasks_processing.go:74] worker 55 stopped. I0420 09:02:22.247643 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0420 09:02:22.247651 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 111.859619ms to process 1 records I0420 09:02:22.270891 1 tasks_processing.go:74] worker 21 stopped. I0420 09:02:22.270908 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 137.736784ms to process 0 records I0420 09:02:22.272908 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0420 09:02:22.277055 1 base_controller.go:82] Caches are synced for LoggingSyncer I0420 09:02:22.277069 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0420 09:02:22.303079 1 cluster_transfer.go:102] no available accepted cluster transfer I0420 09:02:22.303092 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0420 09:02:22.528366 1 sca.go:266] etc-pki-entitlement secret successfully created I0420 09:02:22.528378 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated W0420 09:02:23.170697 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0420 09:02:23.206539 1 tasks_processing.go:74] worker 37 stopped. I0420 09:02:23.206561 1 recorder.go:75] Recording config/silenced_alerts.json with fingerprint=37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570 I0420 09:02:23.206577 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 1.071180841s to process 1 records I0420 09:02:23.206624 1 recorder.go:75] Recording config/alerts with fingerprint=72eeac5fb4c4f3c8d7ff8258a2c51736a2eaad2d6b7f7f72d214cba20b8548d4 I0420 09:02:23.206638 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 1.071156143s to process 1 records I0420 09:02:23.206647 1 tasks_processing.go:74] worker 35 stopped. W0420 09:02:24.170378 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0420 09:02:24.603276 1 tasks_processing.go:74] worker 40 stopped. I0420 09:02:24.603367 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=c847527f6f7a747c52f67be7c58dd96c3ad347c6921e4a32486e968d30931fa4 I0420 09:02:24.603443 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0420 09:02:24.603472 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=d17707e4064e2123df93f3bdda1ea907af0065915f529f23474e859ec9e274af I0420 09:02:24.603482 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0420 09:02:24.603510 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=dd988fc83b35e4783a7a7f577b23138cb5851afeddc8f83828522481ac08040e I0420 09:02:24.603528 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0420 09:02:24.603548 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=9f627a6f6470c1c0f0b669360005c22eeea465ec3dc77a54b3de019d7d79d42d I0420 09:02:24.603582 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=559d26f2508ec8a07b941f9cae1cf452ae4d57b60c8f5994c075fb3adc50e4e9 I0420 09:02:24.603610 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=e3cd27cc4a20892e5bb086b8889afcb7d15f624a8517f82a1a56cdb52e7b4930 I0420 09:02:24.603648 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=d8f113d18152037d779e01833d91f3d14a23b2d0c7d4e94edc9f3604df158f16 I0420 09:02:24.603994 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=d1fbc76f7113ab4c663cb805f473779dfb9e15a5813d5467554f6d3b5a2f5e18 I0420 09:02:24.604021 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=20065559032a529382c4a458b14d32cd303318ce5b3e9b99bbcc1ecc05365fe5 I0420 09:02:24.604031 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0420 09:02:24.604054 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=8d7b7950ac0e0766aeacdf6425d7a8a3c996bad5bcbaa8e19e60c4d3d01fd41b I0420 09:02:24.604066 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0420 09:02:24.604083 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=c657190f749b0d5f31aa3d8e550bd0112c1979e111f090535851d25d6a643206 I0420 09:02:24.604093 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0420 09:02:24.604135 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=0a182b029f8e20f1c982d0e9e1fd858cd25015d7f6366e5fe9c25ad6ad62752f I0420 09:02:24.604148 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0420 09:02:24.604168 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=af96b2a32df261f25b64a0b36a23e8ceed6a638f27620c9fab4c63dcec9260af I0420 09:02:24.604283 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=236edd285612d452b8d2bcfe24bf10329321f797ad75f7947402f639e7b504e7 I0420 09:02:24.604294 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0420 09:02:24.604302 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0420 09:02:24.604324 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0420 09:02:24.604348 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=ccba0e20c0b63adf7103360d9162a576344c2a98ad88ebc0ad7bb223ff45d2f0 I0420 09:02:24.604375 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=4c21d6492e3d721927fb00be2286afed2d4829401befbfa2c0ad20efa75a6d15 I0420 09:02:24.604385 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0420 09:02:24.604401 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=7ffc42e92ba0da137945518f4b85c54067bbdd7b5486c06c031c67462d48f746 I0420 09:02:24.604411 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0420 09:02:24.604431 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=85f7de13e585254413d0e7cca83e6508179b76add0cea1d70edb4819ad929a01 I0420 09:02:24.604444 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0420 09:02:24.604457 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=eb1e148295d99a172b298ba462b9bd9962f9a354689f4bb74921db61654e8762 I0420 09:02:24.604474 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=e187ec188cc1b8afc09a1f910b3489db031c9258167857d826052b260f9af9fb I0420 09:02:24.604488 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=274290166ee40659df35a411b81f723f6d6b615ef88d97025cd15bcc4b725b8a I0420 09:02:24.604519 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=305ce5e1a80611b1be066b8c76165ae5ad33454f9de9112f91b184267369a5de I0420 09:02:24.604531 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0420 09:02:24.604558 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=0c9e3f96270a13bf62715fe3188ae213eeaec7a12f2f6e5105b4e6ae2439a066 I0420 09:02:24.604576 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0420 09:02:24.604589 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0420 09:02:24.604598 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.468594517s to process 39 records W0420 09:02:25.169868 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0420 09:02:26.170454 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0420 09:02:27.170212 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0420 09:02:27.170240 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0420 09:02:27.170256 1 tasks_processing.go:74] worker 19 stopped. E0420 09:02:27.170270 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0420 09:02:27.170281 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0420 09:02:27.170292 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.037170394s to process 1 records I0420 09:02:34.976538 1 tasks_processing.go:74] worker 23 stopped. I0420 09:02:34.976586 1 recorder.go:75] Recording config/installplans with fingerprint=f17dbfacc3bfddf27ca3b213b39495434cd4c4e9e3dbd69566ffb3845bbcf539 I0420 09:02:34.976598 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.843471694s to process 1 records I0420 09:02:35.742267 1 tasks_processing.go:74] worker 18 stopped. I0420 09:02:35.742610 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=4aff5860ec073ddfdf1bec96303c96ef7753a0d476f8315de0a2062bde3a2fd5 I0420 09:02:35.742629 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.609230848s to process 1 records E0420 09:02:35.742700 1 periodic.go:247] "Unhandled Error" err="clusterconfig failed after 13.61s with: function \"tsdb_status\" failed with an error, function \"metrics\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machines\" failed with an error, function \"support_secret\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0420 09:02:35.743817 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "tsdb_status" failed with an error, function "metrics" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machines" failed with an error, function "support_secret" failed with an error, function "machine_healthchecks" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0420 09:02:35.743836 1 periodic.go:209] Running workloads gatherer I0420 09:02:35.743876 1 tasks_processing.go:45] number of workers: 2 I0420 09:02:35.743892 1 tasks_processing.go:69] worker 1 listening for tasks. I0420 09:02:35.743900 1 tasks_processing.go:71] worker 1 working on workload_info task. I0420 09:02:35.743906 1 tasks_processing.go:69] worker 0 listening for tasks. I0420 09:02:35.743920 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0420 09:02:35.750696 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-132.ec2.internal... I0420 09:02:35.750706 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-147.ec2.internal... I0420 09:02:35.750717 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-82.ec2.internal... I0420 09:02:35.772836 1 tasks_processing.go:74] worker 0 stopped. I0420 09:02:35.772854 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 28.902717ms to process 0 records I0420 09:02:40.290606 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 5s I0420 09:02:40.327247 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0420 09:02:40.334932 1 gather_workloads_info.go:387] No image sha256:2904a78e2eb73fd6a9bb94c105c2a056831fb4113fbb7b0607c50adc9d879c9b (9ms) I0420 09:02:40.343081 1 gather_workloads_info.go:387] No image sha256:04c87c054a3f366a7dfbe0a93ebb0c80a098ee16842c5794b67c1202eec61996 (8ms) I0420 09:02:40.352733 1 gather_workloads_info.go:387] No image sha256:765f0d23b637f685f98a31bd47c131b03cf72a40761a3f9a9d6320faa3c33733 (10ms) I0420 09:02:40.361158 1 gather_workloads_info.go:387] No image sha256:2e57e192c3c1240fd935dcd55c8fde5e70e78bf81d6176c96edf21fafe59f8ba (8ms) I0420 09:02:40.369432 1 gather_workloads_info.go:387] No image sha256:1477f3287cc75352305274f248f781c1b7c2a8696f4f7bc450c45a4a5f5b10ea (8ms) I0420 09:02:40.377252 1 gather_workloads_info.go:387] No image sha256:7adc1eab05d6724c76ba751f6df816b08d6e70b78dee9eb94fa6fd9690542c98 (8ms) I0420 09:02:40.384708 1 gather_workloads_info.go:387] No image sha256:101993f6e65d6fda8b90c583b718b720809182961f69bc777295f47e6df4d7ce (7ms) I0420 09:02:40.392904 1 gather_workloads_info.go:387] No image sha256:47154813651033d59751fb655a384dbffb64dd26f10bd7f3be0c3128d0486356 (8ms) I0420 09:02:40.400978 1 gather_workloads_info.go:387] No image sha256:289816958633a763a72dbc44e1dad40466223164e7e253039514f0d974ea5d21 (8ms) I0420 09:02:40.410789 1 gather_workloads_info.go:387] No image sha256:56a85660a445eced5c79a595a0eccf590087c5672d50f49d4c25ad52f9a44f04 (10ms) I0420 09:02:40.435340 1 gather_workloads_info.go:387] No image sha256:03cf4cd7ef1518610c6c7b3ad27d1622d82e98e3dc6e3f8e5d0fceb5c8d3786e (25ms) I0420 09:02:40.534927 1 gather_workloads_info.go:387] No image sha256:50197f22710766515f67944a779e00dd9ae3d17b18732d7324a970353b11f292 (100ms) I0420 09:02:40.635034 1 gather_workloads_info.go:387] No image sha256:943018739e3db1763c3184b460dbc409e058abbac76d57b9927faad317be85e4 (100ms) I0420 09:02:40.735584 1 gather_workloads_info.go:387] No image sha256:521712486e2c6e3c020dad6a1cb340db8e55665b69f7c208fab9cd9e965fd588 (101ms) I0420 09:02:40.835225 1 gather_workloads_info.go:387] No image sha256:513b1166e376feca472fdd5174f6abf0abf02925d6a21b7b4d89a2ff2fc67bed (100ms) I0420 09:02:40.935160 1 gather_workloads_info.go:387] No image sha256:0092c6ff139f8698c91349d507eb3182bc5d6c4ae040f672e347ff504accdea2 (100ms) I0420 09:02:41.034986 1 gather_workloads_info.go:398] Found image sha256:b3909bf664c77097f75b3768830863d642eed3815dab2bfb4415c771ca2d5007 (100ms) I0420 09:02:41.135236 1 gather_workloads_info.go:387] No image sha256:25bda9d34e23ba6bc9df5cf7104bdb237269740a3a76ee025f88b34e68b7e2b5 (100ms) I0420 09:02:41.235258 1 gather_workloads_info.go:387] No image sha256:1a2532940843248c57d52141185dd71fbc393ab28b65d48f682038632c1dbbad (100ms) I0420 09:02:41.335068 1 gather_workloads_info.go:387] No image sha256:b99613494e9d6a4a8207323872d63cc02b830fc80a8bf972e62c64381aa4cd17 (100ms) I0420 09:02:41.434612 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (100ms) I0420 09:02:41.535176 1 gather_workloads_info.go:387] No image sha256:76cc47f3775560477ec548c7b6fddbd99112ad6604f790697eef546576051fb7 (101ms) I0420 09:02:41.635378 1 gather_workloads_info.go:387] No image sha256:09e27b3a67ebeda04d3e597e0561bfb40550de78bdecb53f2f9ff3dfb1cfe7ea (100ms) I0420 09:02:41.734971 1 gather_workloads_info.go:387] No image sha256:7b31223098f08328f5ddea8e5b871dbbd5f5a61ec550e8956f66793c0c6031a9 (100ms) I0420 09:02:41.835391 1 gather_workloads_info.go:387] No image sha256:653c666f842c13e0baae2e89a9b1efe0e2ef56f621ffb5b32005115d2a26ab8c (100ms) I0420 09:02:41.935123 1 gather_workloads_info.go:387] No image sha256:5a95c19d82767e0235b4edb4a0536482c816904897aae1dc3eb255cb52b87a9f (100ms) I0420 09:02:42.034885 1 gather_workloads_info.go:387] No image sha256:3612c5a433effd17b7aff7ff6edf4fb27b61153908e3a0294158e4886a1d600d (100ms) I0420 09:02:42.135816 1 gather_workloads_info.go:387] No image sha256:0a99240166165eb5718e7516a43282fe32df9c7c5e809b31b58abe44e42ff94d (101ms) I0420 09:02:42.235976 1 gather_workloads_info.go:387] No image sha256:2598489729a4b258e4ecda4a06f6875133f2a10ced5c5241f8a57a8a05418e36 (100ms) I0420 09:02:42.335691 1 gather_workloads_info.go:387] No image sha256:2e564f336c77116053f34d4201d364d8da04e789cfffa0ea422574c95f2d6404 (100ms) I0420 09:02:42.435666 1 gather_workloads_info.go:387] No image sha256:a56211d075aa43cbb491f669a5b2e46ee023dc95b7d51dbac28f463948c5ad61 (100ms) I0420 09:02:42.535204 1 gather_workloads_info.go:387] No image sha256:a0105d1eb62cf6ac9e5e2ef28d3e89bf6dc514bc594fc7090fe5a5ee18a09c87 (100ms) I0420 09:02:42.635097 1 gather_workloads_info.go:387] No image sha256:7bd3e88d6d3f878a89203df590b684c5a77bf08df4717451d3197ab74683a80e (100ms) I0420 09:02:42.735292 1 gather_workloads_info.go:387] No image sha256:90bdf9e678fe18e35401e34e1cb0912d822e9cb32b9e41cc545870540e9a87f2 (100ms) I0420 09:02:42.835511 1 gather_workloads_info.go:387] No image sha256:3d6ea9e1633f38070718a1357fcb80e6cb3932aafcc4d65e659795b186c5da15 (100ms) I0420 09:02:42.935425 1 gather_workloads_info.go:387] No image sha256:74713675d580a47a56b628469c4c05c41e114e0b935960d718ff0cf7e72f613d (100ms) I0420 09:02:43.035237 1 gather_workloads_info.go:387] No image sha256:c940ea87e7d133d75ba0002ef00c0806825eed3db8094cdb260d1bac18127733 (100ms) I0420 09:02:43.135441 1 gather_workloads_info.go:387] No image sha256:695cf2f0cc07683c2a3ce1eaf3e56fe18abc6e2bac716f7d9843f5d173b9df52 (100ms) I0420 09:02:43.235953 1 gather_workloads_info.go:387] No image sha256:6e5df2aab9ce759e8f957341a11c087c233307e3c8f8760d21270efc19ccd3ce (100ms) I0420 09:02:43.335457 1 gather_workloads_info.go:387] No image sha256:4556896f77307821531ef91b7b7faccb82b824ea695693b2989f597f0deca038 (99ms) I0420 09:02:43.435173 1 gather_workloads_info.go:387] No image sha256:0af73ad7a47c9ff53269ae2bc5163bf9e389085220c34e4bd42c92c4e7a85f6d (100ms) I0420 09:02:43.534356 1 gather_workloads_info.go:387] No image sha256:91828234f107c068c8a4966d08370ae7b73e637651dbc6d92c18c4553402c22c (99ms) I0420 09:02:43.635283 1 gather_workloads_info.go:387] No image sha256:5f0b67cfbbc381243fb91ccc17345b56d05f4d717c667e8c644e5bf05633ba71 (101ms) I0420 09:02:43.735264 1 gather_workloads_info.go:387] No image sha256:a498046d64605bcccee2440aa4f04a4602baaae263cf01d977ec5208e876b1fd (100ms) I0420 09:02:43.834936 1 gather_workloads_info.go:398] Found image sha256:32d4561d4bf1b3c4e3a7878f918f9c1422bab85bdf8711fbdf905d59692fee81 (100ms) I0420 09:02:43.936068 1 gather_workloads_info.go:387] No image sha256:a258c226562adb14e3a163a1940938526ee6a0928982a7667d85d9a7334ce639 (101ms) I0420 09:02:44.034909 1 gather_workloads_info.go:387] No image sha256:1e65f9f5490fbc930d2dce96f03ddcb28990b839d7dc011ef5de5bec6c5095bf (99ms) I0420 09:02:44.135123 1 gather_workloads_info.go:387] No image sha256:36b9e89c3cfcf1ab9ae500486e38afb6862cba48cb0b4d84a09508ab8f3d299f (100ms) I0420 09:02:44.135158 1 tasks_processing.go:74] worker 1 stopped. I0420 09:02:44.135482 1 recorder.go:75] Recording config/workload_info with fingerprint=c8d4c1ecc2d828ad7f08077205d53f61ae987518522df45761a27bb31da30cae I0420 09:02:44.135501 1 gather.go:177] gatherer "workloads" function "workload_info" took 8.391249273s to process 1 records I0420 09:02:44.135519 1 periodic.go:256] Periodic gather workloads completed in 8.391s I0420 09:02:44.135528 1 controllerstatus.go:80] name=periodic-workloads healthy=true reason= message= I0420 09:02:44.135533 1 periodic.go:209] Running conditional gatherer I0420 09:02:44.141744 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules I0420 09:02:44.240964 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer I0420 09:02:45.296422 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0420 09:02:45.302269 1 conditional_gatherer.go:392] cluster version is '4.20.8' I0420 09:02:45.302300 1 tasks_processing.go:45] number of workers: 3 I0420 09:02:45.302311 1 tasks_processing.go:69] worker 2 listening for tasks. I0420 09:02:45.302315 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0420 09:02:45.302330 1 tasks_processing.go:69] worker 1 listening for tasks. I0420 09:02:45.302327 1 tasks_processing.go:69] worker 0 listening for tasks. I0420 09:02:45.302343 1 tasks_processing.go:71] worker 1 working on remote_configuration task. I0420 09:02:45.302346 1 tasks_processing.go:74] worker 0 stopped. I0420 09:02:45.302363 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0420 09:02:45.302434 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=c8d4892cd4f2eea27d57353c6055eeb530bc2d811bfe041f6984d90395255a8c I0420 09:02:45.302454 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.296µs to process 1 records I0420 09:02:45.302503 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0420 09:02:45.302514 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.459µs to process 1 records I0420 09:02:45.302523 1 tasks_processing.go:74] worker 1 stopped. I0420 09:02:45.302683 1 tasks_processing.go:74] worker 2 stopped. I0420 09:02:45.302706 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 299.888µs to process 0 records I0420 09:02:45.302744 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0420 09:02:45.302765 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0420 09:02:45.329581 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=fa2d952290e41ed413d211fbee34a37c0e4bd69bd5873af9ea50fe6e45996311 I0420 09:02:45.329719 1 diskrecorder.go:70] Writing 111 records to /var/lib/insights-operator/insights-2026-04-20-090245.tar.gz I0420 09:02:45.336077 1 diskrecorder.go:51] Wrote 111 records to disk in 6ms I0420 09:02:45.336131 1 periodic.go:278] Gathering cluster info every 2h0m0s I0420 09:02:45.336152 1 periodic.go:279] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0420 09:03:52.133678 1 diskrecorder.go:223] Found files to send: insights-2026-04-20-090245.tar.gz I0420 09:03:52.133709 1 insightsuploader.go:150] Checking archives to upload periodically every 15m5.346191148s I0420 09:03:52.133719 1 insightsuploader.go:165] Uploading latest report since 2026-04-20T09:02:15Z I0420 09:03:52.144415 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0420 09:03:52.352959 1 requests.go:87] Successfully reported id=2026-04-20T09:03:52Z x-rh-insights-request-id=05fb692af2164ab8bb3ee972e011778d, wrote=49252 I0420 09:03:52.353019 1 insightsuploader.go:187] Uploaded report successfully in 219.293984ms I0420 09:03:52.353056 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0420 09:03:52.353072 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0420 09:03:52.353081 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0420 09:03:52.359405 1 controller.go:512] The operator is healthy I0420 09:04:22.184445 1 controller.go:512] The operator is healthy I0420 09:04:54.127008 1 insightsreport.go:137] Pulling report from smart-proxy I0420 09:04:54.127033 1 insightsreport.go:149] Retrieving report I0420 09:04:54.134318 1 requests.go:110] Retrieving report for cluster: bcc5814d-c2c5-4e0e-ae94-fe0e9f22c383 I0420 09:04:54.134330 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/bcc5814d-c2c5-4e0e-ae94-fe0e9f22c383/reports I0420 09:04:54.137428 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/bcc5814d-c2c5-4e0e-ae94-fe0e9f22c383/reports I0420 09:04:54.445912 1 insightsreport.go:184] Report retrieved I0420 09:04:54.472823 1 insightsreport.go:239] Report retrieved correctly I0420 09:06:22.183927 1 controller.go:512] The operator is healthy I0420 09:07:22.125959 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 09:07:22.135267 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 09:07:22.135287 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 09:07:22.140245 1 secretconfigobserver.go:119] support secret does not exist I0420 09:08:22.183829 1 controller.go:512] The operator is healthy I0420 09:10:22.183314 1 controller.go:512] The operator is healthy I0420 09:12:22.123046 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 09:12:22.141090 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 09:12:22.146845 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 09:12:22.146860 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 09:12:22.151733 1 secretconfigobserver.go:119] support secret does not exist I0420 09:12:22.182895 1 controller.go:512] The operator is healthy I0420 09:14:22.183797 1 controller.go:512] The operator is healthy I0420 09:16:22.183778 1 controller.go:512] The operator is healthy I0420 09:17:22.152431 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 09:17:22.158470 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 09:17:22.158489 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 09:17:22.162913 1 secretconfigobserver.go:119] support secret does not exist I0420 09:18:22.184056 1 controller.go:512] The operator is healthy I0420 09:19:02.700204 1 insightsuploader.go:146] Nothing to report since 2026-04-20T09:03:52Z I0420 09:20:22.184177 1 controller.go:512] The operator is healthy I0420 09:22:22.123064 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 09:22:22.163191 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 09:22:22.168301 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 09:22:22.168317 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 09:22:22.172682 1 secretconfigobserver.go:119] support secret does not exist I0420 09:22:22.183190 1 controller.go:512] The operator is healthy I0420 09:24:22.183918 1 controller.go:512] The operator is healthy I0420 09:26:22.185568 1 controller.go:512] The operator is healthy I0420 09:27:22.173311 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 09:27:22.179394 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 09:27:22.179417 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 09:27:22.183867 1 secretconfigobserver.go:119] support secret does not exist I0420 09:28:22.184058 1 controller.go:512] The operator is healthy I0420 09:30:22.184272 1 controller.go:512] The operator is healthy