I0410 16:39:32.226591 1 cmd.go:253] Using service-serving-cert provided certificates I0410 16:39:32.227014 1 observer_polling.go:159] Starting file observer I0410 16:39:32.589263 1 operator.go:60] Starting insights-operator v0.0.0-master+$Format:%H$ I0410 16:39:32.589467 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0410 16:39:32.590012 1 secure_serving.go:57] Forcing use of http/1.1 only W0410 16:39:32.590037 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0410 16:39:32.590043 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0410 16:39:32.590051 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. I0410 16:39:32.590034 1 simple_featuregate_reader.go:171] Starting feature-gate-detector W0410 16:39:32.590055 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0410 16:39:32.590066 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0410 16:39:32.590070 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0410 16:39:32.593572 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"792284c8-639e-45b2-956c-90b0f13cef77", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}} I0410 16:39:32.593568 1 operator.go:125] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GatewayAPI GatewayAPIController HighlyAvailableArbiter ImageVolume IngressControllerLBSubnetsAWS KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController SigstoreImageVerification StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPClusterHostedDNSInstall GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall ImageModeStatusReporting ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesAzure ManagedBootImagesvSphere MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutatingAdmissionPolicy NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterOperations NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PreconfiguredUDNAddresses SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeAttributesClass VolumeGroupSnapshot] I0410 16:39:32.595365 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0410 16:39:32.595366 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0410 16:39:32.595403 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0410 16:39:32.595418 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0410 16:39:32.595340 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0410 16:39:32.595438 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0410 16:39:32.595684 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0410 16:39:32.595895 1 secure_serving.go:213] Serving securely on [::]:8443 I0410 16:39:32.595920 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0410 16:39:32.601732 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0410 16:39:32.601754 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0410 16:39:32.601852 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0410 16:39:32.606422 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0410 16:39:32.606442 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0410 16:39:32.610698 1 secretconfigobserver.go:119] support secret does not exist I0410 16:39:32.615138 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0410 16:39:32.619674 1 secretconfigobserver.go:119] support secret does not exist I0410 16:39:32.622027 1 recorder.go:161] Pruning old reports every 4h26m36s, max age is 288h0m0s I0410 16:39:32.626880 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0410 16:39:32.626897 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0410 16:39:32.626902 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0410 16:39:32.626912 1 insightsreport.go:296] Starting report retriever I0410 16:39:32.626917 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0410 16:39:32.626919 1 periodic.go:209] Running clusterconfig gatherer I0410 16:39:32.626957 1 tasks_processing.go:45] number of workers: 64 I0410 16:39:32.626979 1 tasks_processing.go:69] worker 1 listening for tasks. I0410 16:39:32.626987 1 tasks_processing.go:69] worker 0 listening for tasks. I0410 16:39:32.626989 1 tasks_processing.go:69] worker 17 listening for tasks. I0410 16:39:32.626997 1 tasks_processing.go:69] worker 9 listening for tasks. I0410 16:39:32.627000 1 tasks_processing.go:71] worker 17 working on openstack_dataplanedeployments task. I0410 16:39:32.627002 1 tasks_processing.go:71] worker 9 working on sap_pods task. I0410 16:39:32.627007 1 tasks_processing.go:69] worker 18 listening for tasks. I0410 16:39:32.627008 1 tasks_processing.go:69] worker 11 listening for tasks. I0410 16:39:32.627010 1 tasks_processing.go:69] worker 5 listening for tasks. I0410 16:39:32.627002 1 tasks_processing.go:69] worker 39 listening for tasks. I0410 16:39:32.627019 1 tasks_processing.go:69] worker 12 listening for tasks. I0410 16:39:32.627015 1 tasks_processing.go:69] worker 19 listening for tasks. I0410 16:39:32.627019 1 tasks_processing.go:69] worker 20 listening for tasks. I0410 16:39:32.627025 1 tasks_processing.go:69] worker 2 listening for tasks. I0410 16:39:32.627026 1 tasks_processing.go:69] worker 30 listening for tasks. I0410 16:39:32.627031 1 tasks_processing.go:69] worker 29 listening for tasks. I0410 16:39:32.627035 1 tasks_processing.go:69] worker 32 listening for tasks. I0410 16:39:32.627031 1 tasks_processing.go:69] worker 31 listening for tasks. I0410 16:39:32.627037 1 tasks_processing.go:69] worker 4 listening for tasks. I0410 16:39:32.627040 1 tasks_processing.go:69] worker 13 listening for tasks. I0410 16:39:32.627031 1 tasks_processing.go:69] worker 3 listening for tasks. I0410 16:39:32.627043 1 tasks_processing.go:69] worker 21 listening for tasks. I0410 16:39:32.627044 1 tasks_processing.go:69] worker 33 listening for tasks. I0410 16:39:32.627049 1 tasks_processing.go:69] worker 16 listening for tasks. I0410 16:39:32.627050 1 tasks_processing.go:69] worker 34 listening for tasks. I0410 16:39:32.627053 1 tasks_processing.go:69] worker 22 listening for tasks. I0410 16:39:32.627057 1 tasks_processing.go:69] worker 52 listening for tasks. I0410 16:39:32.627056 1 tasks_processing.go:69] worker 35 listening for tasks. I0410 16:39:32.627061 1 tasks_processing.go:69] worker 23 listening for tasks. I0410 16:39:32.627062 1 tasks_processing.go:69] worker 14 listening for tasks. I0410 16:39:32.627052 1 tasks_processing.go:69] worker 15 listening for tasks. I0410 16:39:32.627071 1 tasks_processing.go:69] worker 50 listening for tasks. I0410 16:39:32.627062 1 tasks_processing.go:69] worker 36 listening for tasks. I0410 16:39:32.627075 1 tasks_processing.go:69] worker 27 listening for tasks. I0410 16:39:32.627077 1 tasks_processing.go:69] worker 51 listening for tasks. I0410 16:39:32.627077 1 tasks_processing.go:69] worker 38 listening for tasks. I0410 16:39:32.627080 1 tasks_processing.go:69] worker 47 listening for tasks. I0410 16:39:32.627084 1 tasks_processing.go:69] worker 25 listening for tasks. I0410 16:39:32.627085 1 tasks_processing.go:69] worker 28 listening for tasks. I0410 16:39:32.627084 1 tasks_processing.go:69] worker 41 listening for tasks. I0410 16:39:32.627089 1 tasks_processing.go:69] worker 43 listening for tasks. I0410 16:39:32.627092 1 tasks_processing.go:69] worker 44 listening for tasks. I0410 16:39:32.627094 1 tasks_processing.go:69] worker 42 listening for tasks. I0410 16:39:32.627095 1 tasks_processing.go:69] worker 6 listening for tasks. I0410 16:39:32.627065 1 tasks_processing.go:69] worker 48 listening for tasks. I0410 16:39:32.627101 1 tasks_processing.go:69] worker 61 listening for tasks. I0410 16:39:32.627095 1 tasks_processing.go:69] worker 26 listening for tasks. I0410 16:39:32.627101 1 tasks_processing.go:69] worker 59 listening for tasks. I0410 16:39:32.627107 1 tasks_processing.go:69] worker 53 listening for tasks. I0410 16:39:32.627112 1 tasks_processing.go:69] worker 56 listening for tasks. I0410 16:39:32.627110 1 tasks_processing.go:71] worker 61 working on openstack_dataplanenodesets task. I0410 16:39:32.627115 1 tasks_processing.go:71] worker 29 working on service_accounts task. I0410 16:39:32.627118 1 tasks_processing.go:71] worker 13 working on dvo_metrics task. I0410 16:39:32.627119 1 tasks_processing.go:71] worker 35 working on mutating_webhook_configurations task. I0410 16:39:32.627121 1 tasks_processing.go:71] worker 56 working on container_images task. I0410 16:39:32.627122 1 tasks_processing.go:71] worker 21 working on jaegers task. I0410 16:39:32.627124 1 tasks_processing.go:71] worker 16 working on proxies task. I0410 16:39:32.627126 1 tasks_processing.go:71] worker 59 working on infrastructures task. I0410 16:39:32.627123 1 tasks_processing.go:69] worker 55 listening for tasks. I0410 16:39:32.627128 1 tasks_processing.go:71] worker 1 working on schedulers task. I0410 16:39:32.627141 1 tasks_processing.go:71] worker 55 working on crds task. I0410 16:39:32.627155 1 tasks_processing.go:71] worker 53 working on aggregated_monitoring_cr_names task. I0410 16:39:32.627167 1 tasks_processing.go:71] worker 12 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0410 16:39:32.627233 1 tasks_processing.go:71] worker 0 working on sap_config task. I0410 16:39:32.627441 1 tasks_processing.go:71] worker 31 working on storage_cluster task. I0410 16:39:32.627462 1 tasks_processing.go:71] worker 32 working on storage_classes task. I0410 16:39:32.627608 1 tasks_processing.go:71] worker 36 working on machines task. I0410 16:39:32.627619 1 tasks_processing.go:71] worker 27 working on container_runtime_configs task. I0410 16:39:32.627625 1 tasks_processing.go:71] worker 51 working on metrics task. I0410 16:39:32.627630 1 tasks_processing.go:71] worker 41 working on config_maps task. I0410 16:39:32.627629 1 tasks_processing.go:71] worker 39 working on nodenetworkconfigurationpolicies task. I0410 16:39:32.627636 1 tasks_processing.go:71] worker 47 working on active_alerts task. I0410 16:39:32.627638 1 tasks_processing.go:71] worker 4 working on qemu_kubevirt_launcher_logs task. I0410 16:39:32.627645 1 tasks_processing.go:69] worker 54 listening for tasks. I0410 16:39:32.627650 1 tasks_processing.go:71] worker 11 working on openshift_logging task. I0410 16:39:32.627654 1 tasks_processing.go:71] worker 54 working on validating_webhook_configurations task. I0410 16:39:32.627659 1 tasks_processing.go:71] worker 2 working on operators task. I0410 16:39:32.627662 1 tasks_processing.go:71] worker 25 working on image_registries task. I0410 16:39:32.627111 1 tasks_processing.go:71] worker 22 working on support_secret task. I0410 16:39:32.627706 1 tasks_processing.go:71] worker 28 working on machine_config_pools task. I0410 16:39:32.627721 1 tasks_processing.go:71] worker 44 working on machine_autoscalers task. I0410 16:39:32.627725 1 tasks_processing.go:71] worker 43 working on pdbs task. I0410 16:39:32.627754 1 tasks_processing.go:71] worker 30 working on version task. I0410 16:39:32.627869 1 tasks_processing.go:71] worker 42 working on cost_management_metrics_configs task. I0410 16:39:32.627119 1 tasks_processing.go:71] worker 33 working on overlapping_namespace_uids task. I0410 16:39:32.627069 1 tasks_processing.go:69] worker 24 listening for tasks. I0410 16:39:32.627065 1 tasks_processing.go:69] worker 49 listening for tasks. I0410 16:39:32.628652 1 tasks_processing.go:71] worker 49 working on lokistack task. I0410 16:39:32.627601 1 tasks_processing.go:71] worker 15 working on tsdb_status task. I0410 16:39:32.629007 1 tasks_processing.go:71] worker 19 working on clusterroles task. I0410 16:39:32.627604 1 tasks_processing.go:71] worker 50 working on image_pruners task. I0410 16:39:32.628681 1 tasks_processing.go:71] worker 20 working on node_logs task. I0410 16:39:32.627071 1 tasks_processing.go:69] worker 46 listening for tasks. I0410 16:39:32.629602 1 tasks_processing.go:71] worker 46 working on machine_configs task. I0410 16:39:32.627077 1 tasks_processing.go:69] worker 40 listening for tasks. I0410 16:39:32.627086 1 tasks_processing.go:69] worker 7 listening for tasks. I0410 16:39:32.627090 1 tasks_processing.go:69] worker 8 listening for tasks. I0410 16:39:32.627098 1 tasks_processing.go:69] worker 62 listening for tasks. I0410 16:39:32.627099 1 tasks_processing.go:69] worker 60 listening for tasks. I0410 16:39:32.627104 1 tasks_processing.go:71] worker 6 working on networks task. I0410 16:39:32.627105 1 tasks_processing.go:71] worker 48 working on image task. I0410 16:39:32.627105 1 tasks_processing.go:69] worker 57 listening for tasks. I0410 16:39:32.627108 1 tasks_processing.go:71] worker 18 working on authentication task. I0410 16:39:32.627108 1 tasks_processing.go:69] worker 58 listening for tasks. I0410 16:39:32.627112 1 tasks_processing.go:71] worker 34 working on nodenetworkstates task. I0410 16:39:32.627112 1 tasks_processing.go:71] worker 26 working on machine_sets task. I0410 16:39:32.627116 1 tasks_processing.go:71] worker 52 working on operators_pods_and_events task. I0410 16:39:32.627118 1 tasks_processing.go:71] worker 3 working on ceph_cluster task. I0410 16:39:32.627118 1 tasks_processing.go:69] worker 63 listening for tasks. I0410 16:39:32.627590 1 tasks_processing.go:71] worker 38 working on sap_datahubs task. I0410 16:39:32.627594 1 tasks_processing.go:71] worker 23 working on machine_healthchecks task. I0410 16:39:32.627001 1 tasks_processing.go:69] worker 10 listening for tasks. I0410 16:39:32.627597 1 tasks_processing.go:71] worker 14 working on monitoring_persistent_volumes task. I0410 16:39:32.627068 1 tasks_processing.go:69] worker 45 listening for tasks. I0410 16:39:32.628952 1 tasks_processing.go:71] worker 24 working on olm_operators task. I0410 16:39:32.627069 1 tasks_processing.go:69] worker 37 listening for tasks. I0410 16:39:32.630069 1 tasks_processing.go:71] worker 40 working on pod_network_connectivity_checks task. I0410 16:39:32.630896 1 tasks_processing.go:71] worker 7 working on silenced_alerts task. I0410 16:39:32.630907 1 tasks_processing.go:71] worker 8 working on openstack_controlplanes task. I0410 16:39:32.631081 1 tasks_processing.go:71] worker 62 working on certificate_signing_requests task. I0410 16:39:32.631107 1 tasks_processing.go:71] worker 60 working on nodes task. I0410 16:39:32.631155 1 tasks_processing.go:71] worker 57 working on openstack_version task. I0410 16:39:32.631258 1 tasks_processing.go:71] worker 58 working on ingress task. I0410 16:39:32.631538 1 tasks_processing.go:71] worker 63 working on oauths task. I0410 16:39:32.632426 1 tasks_processing.go:71] worker 5 working on cluster_apiserver task. I0410 16:39:32.632719 1 tasks_processing.go:71] worker 10 working on feature_gates task. I0410 16:39:32.632902 1 tasks_processing.go:71] worker 45 working on openshift_machine_api_events task. I0410 16:39:32.632954 1 tasks_processing.go:71] worker 37 working on install_plans task. I0410 16:39:32.633472 1 tasks_processing.go:71] worker 21 working on ingress_certificates task. I0410 16:39:32.633548 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 6.340152ms to process 0 records I0410 16:39:32.633560 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 6.255608ms to process 0 records I0410 16:39:32.633567 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 6.504489ms to process 0 records I0410 16:39:32.633573 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 4.562854ms to process 0 records I0410 16:39:32.633624 1 tasks_processing.go:74] worker 49 stopped. I0410 16:39:32.633638 1 tasks_processing.go:74] worker 17 stopped. I0410 16:39:32.633643 1 tasks_processing.go:74] worker 0 stopped. I0410 16:39:32.633936 1 controller.go:128] Initializing last reported time to 0001-01-01T00:00:00Z I0410 16:39:32.633957 1 controller.go:326] The initial operator extension status is healthy I0410 16:39:32.633967 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0410 16:39:32.633974 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0410 16:39:32.633978 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0410 16:39:32.633999 1 controller.go:489] The operator is still being initialized I0410 16:39:32.634010 1 controller.go:512] The operator is healthy I0410 16:39:32.634088 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0410 16:39:32.634091 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0410 16:39:32.634241 1 operator.go:288] started I0410 16:39:32.634270 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0410 16:39:32.635477 1 tasks_processing.go:74] worker 59 stopped. I0410 16:39:32.636023 1 recorder.go:75] Recording config/infrastructure with fingerprint=9099832542006c4502937d05888d5f72b71583529d7d3f5fce8b9229cb22f2a6 I0410 16:39:32.636039 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 8.335883ms to process 1 records I0410 16:39:32.636047 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 8.28617ms to process 0 records I0410 16:39:32.636062 1 tasks_processing.go:74] worker 31 stopped. I0410 16:39:32.636101 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=7e3b529336fd2b7a60a8882580a81d7233132ce350982fe3a38760e6b8ae09c8 I0410 16:39:32.636110 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 8.833735ms to process 1 records I0410 16:39:32.636116 1 tasks_processing.go:74] worker 1 stopped. E0410 16:39:32.640315 1 gather_active_alerts.go:64] Unable to retrieve most recent alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp: lookup alertmanager-main.openshift-monitoring.svc on 172.30.0.10:53: no such host I0410 16:39:32.640330 1 tasks_processing.go:74] worker 47 stopped. E0410 16:39:32.640337 1 gather.go:140] gatherer "clusterconfig" function "active_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp: lookup alertmanager-main.openshift-monitoring.svc on 172.30.0.10:53: no such host I0410 16:39:32.640346 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 12.688324ms to process 0 records E0410 16:39:32.640386 1 gather_silenced_alerts.go:52] Unable to retrieve silenced alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp: lookup alertmanager-main.openshift-monitoring.svc on 172.30.0.10:53: no such host I0410 16:39:32.640420 1 tasks_processing.go:74] worker 7 stopped. E0410 16:39:32.640432 1 gather.go:140] gatherer "clusterconfig" function "silenced_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp: lookup alertmanager-main.openshift-monitoring.svc on 172.30.0.10:53: no such host I0410 16:39:32.640448 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 9.496215ms to process 0 records E0410 16:39:32.641471 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0410 16:39:32.641489 1 tasks_processing.go:74] worker 51 stopped. E0410 16:39:32.641496 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0410 16:39:32.641508 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 13.858853ms to process 0 records E0410 16:39:32.641527 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0410 16:39:32.641545 1 tasks_processing.go:74] worker 15 stopped. E0410 16:39:32.641554 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0410 16:39:32.641564 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 12.550546ms to process 0 records I0410 16:39:32.642961 1 tasks_processing.go:74] worker 27 stopped. I0410 16:39:32.642971 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 15.335932ms to process 0 records E0410 16:39:32.642979 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0410 16:39:32.642985 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 12.888725ms to process 0 records I0410 16:39:32.642992 1 tasks_processing.go:74] worker 40 stopped. I0410 16:39:32.642997 1 tasks_processing.go:74] worker 9 stopped. I0410 16:39:32.643012 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 15.986116ms to process 0 records I0410 16:39:32.643103 1 tasks_processing.go:74] worker 36 stopped. E0410 16:39:32.643113 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0410 16:39:32.643124 1 gather.go:177] gatherer "clusterconfig" function "machines" took 15.482161ms to process 0 records I0410 16:39:32.643215 1 tasks_processing.go:74] worker 16 stopped. I0410 16:39:32.643269 1 recorder.go:75] Recording config/proxy with fingerprint=821a67fade3bc539f9f2a82f69506ce9daad24763ed65ce0d0f0739b302cc08a I0410 16:39:32.643280 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 16.077335ms to process 1 records I0410 16:39:32.643360 1 tasks_processing.go:74] worker 50 stopped. I0410 16:39:32.643492 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=bcef40d1d8e51fcdb7c907d130dc6b095a99f461d89266084e65d75c80474b63 I0410 16:39:32.643501 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 13.89163ms to process 1 records I0410 16:39:32.644035 1 tasks_processing.go:74] worker 8 stopped. I0410 16:39:32.644044 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 13.11852ms to process 0 records W0410 16:39:32.645831 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0410 16:39:32.649868 1 tasks_processing.go:74] worker 57 stopped. I0410 16:39:32.649883 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 18.696871ms to process 0 records I0410 16:39:32.652750 1 gather_logs.go:145] no pods in namespace were found I0410 16:39:32.652763 1 tasks_processing.go:74] worker 4 stopped. I0410 16:39:32.652770 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 25.119908ms to process 0 records I0410 16:39:32.652924 1 tasks_processing.go:74] worker 48 stopped. I0410 16:39:32.653042 1 recorder.go:75] Recording config/image with fingerprint=a8f8ea5a2008831c1c5ba05216785bd69f99c302c9aa5385d88cf41326177dee I0410 16:39:32.653061 1 gather.go:177] gatherer "clusterconfig" function "image" took 23.161413ms to process 1 records I0410 16:39:32.653155 1 tasks_processing.go:74] worker 6 stopped. I0410 16:39:32.653227 1 recorder.go:75] Recording config/network with fingerprint=c0c44af09334472a539db75d37497eb8d195819216e853dc890407aea7f18ef0 I0410 16:39:32.653238 1 gather.go:177] gatherer "clusterconfig" function "networks" took 23.305438ms to process 1 records I0410 16:39:32.657049 1 tasks_processing.go:74] worker 34 stopped. I0410 16:39:32.657060 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 27.266704ms to process 0 records I0410 16:39:32.657088 1 tasks_processing.go:74] worker 20 stopped. I0410 16:39:32.657094 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 27.594756ms to process 0 records I0410 16:39:32.657146 1 tasks_processing.go:74] worker 45 stopped. I0410 16:39:32.657163 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 24.232729ms to process 0 records I0410 16:39:32.657190 1 tasks_processing.go:74] worker 44 stopped. I0410 16:39:32.657200 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 29.464008ms to process 0 records I0410 16:39:32.657209 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 27.417961ms to process 0 records I0410 16:39:32.657215 1 tasks_processing.go:74] worker 26 stopped. I0410 16:39:32.657278 1 tasks_processing.go:74] worker 58 stopped. I0410 16:39:32.657489 1 recorder.go:75] Recording config/ingress with fingerprint=d18dbb86bfef8774a1ae66d19267aa4504514340635148392bf302b893b9debe I0410 16:39:32.657505 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 26.006488ms to process 1 records I0410 16:39:32.657584 1 tasks_processing.go:74] worker 18 stopped. I0410 16:39:32.657727 1 recorder.go:75] Recording config/authentication with fingerprint=5089c40f0608450a34dcfb30c368d298ad7ac8c476fe4f8f1e2cb3b36c32d9ec I0410 16:39:32.657741 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 27.59876ms to process 1 records I0410 16:39:32.657749 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 27.650152ms to process 0 records I0410 16:39:32.657758 1 tasks_processing.go:74] worker 38 stopped. I0410 16:39:32.657790 1 tasks_processing.go:74] worker 25 stopped. I0410 16:39:32.658101 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=b1f243914910015cc724a00bd6c87c884536ac87a548e16959a8e483758538a1 I0410 16:39:32.658117 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 30.121162ms to process 1 records I0410 16:39:32.661736 1 tasks_processing.go:74] worker 39 stopped. I0410 16:39:32.661744 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 34.096043ms to process 0 records I0410 16:39:32.661759 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 31.954537ms to process 0 records I0410 16:39:32.661765 1 tasks_processing.go:74] worker 3 stopped. I0410 16:39:32.661993 1 tasks_processing.go:74] worker 63 stopped. I0410 16:39:32.662161 1 recorder.go:75] Recording config/oauth with fingerprint=928a67ad08f89123b90b8999fcf6cf9ce18beaa1100673040165095d9e8c4595 I0410 16:39:32.662172 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 30.442554ms to process 1 records E0410 16:39:32.662179 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0410 16:39:32.662184 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 32.183484ms to process 0 records I0410 16:39:32.662189 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 34.449764ms to process 0 records I0410 16:39:32.662193 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 35.055458ms to process 0 records I0410 16:39:32.662196 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 34.297636ms to process 0 records I0410 16:39:32.662199 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 34.525788ms to process 0 records E0410 16:39:32.662203 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0410 16:39:32.662208 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 34.510399ms to process 0 records I0410 16:39:32.662213 1 tasks_processing.go:74] worker 22 stopped. I0410 16:39:32.662216 1 tasks_processing.go:74] worker 23 stopped. I0410 16:39:32.662220 1 tasks_processing.go:74] worker 28 stopped. I0410 16:39:32.662223 1 tasks_processing.go:74] worker 61 stopped. I0410 16:39:32.662226 1 tasks_processing.go:74] worker 42 stopped. I0410 16:39:32.662229 1 tasks_processing.go:74] worker 11 stopped. I0410 16:39:32.662242 1 tasks_processing.go:74] worker 32 stopped. I0410 16:39:32.662341 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=1468271acb42d19d23f8281ef71a6e1a3358f26c6852d89575b0d4bd4d8c5969 I0410 16:39:32.662367 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=bd73b4edd9f8da561da115600b0979064e18d935d1fda575d89ae4de9e70f752 I0410 16:39:32.662384 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 34.771596ms to process 2 records I0410 16:39:32.662692 1 tasks_processing.go:74] worker 5 stopped. I0410 16:39:32.662872 1 recorder.go:75] Recording config/apiserver with fingerprint=02baf48dad7125c34686d8ea609f6c862ab1e99de77b169a72a217a473a14655 I0410 16:39:32.662891 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 30.248307ms to process 1 records I0410 16:39:32.662965 1 tasks_processing.go:74] worker 33 stopped. I0410 16:39:32.662990 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0410 16:39:32.662999 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 35.072983ms to process 1 records I0410 16:39:32.663079 1 tasks_processing.go:74] worker 60 stopped. I0410 16:39:32.663352 1 recorder.go:75] Recording config/node/ip-10-0-1-37.ec2.internal with fingerprint=d5627fd56a1069fdb6810f6819902fca288ef4463efd6c8cdf6ba1f3afbc104b I0410 16:39:32.663367 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 31.958325ms to process 1 records I0410 16:39:32.663928 1 tasks_processing.go:74] worker 14 stopped. I0410 16:39:32.663943 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 34.059694ms to process 0 records I0410 16:39:32.663980 1 tasks_processing.go:74] worker 43 stopped. I0410 16:39:32.664061 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=b3c3a9af76077faa1c9296758a225eececaee5e1e6cc23ec117d800c473352a1 I0410 16:39:32.664079 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=a24e9d03f1b7bba8c8172b2c82da6e5144758cc78cb304ccddf9576f3a028b57 I0410 16:39:32.664333 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=73341b86430c6e34e2c76c11bdfb169e6eb4b6178518aa3dcfed7da45b2f083b I0410 16:39:32.664432 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=90875c084f4124e4f7edfddf802301d0057d62d58a1be28fe51d2fd9575f5526 I0410 16:39:32.664470 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=54ff59fd34e3c11c294106e09996e50b8676fc21892081b76c5dbaa9383a0708 I0410 16:39:32.664554 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=c646c52dc46250d77688ea0210ef0238511ef451347b9616ab2978810973acb3 I0410 16:39:32.664587 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 36.24876ms to process 6 records I0410 16:39:32.664617 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 34.156494ms to process 0 records I0410 16:39:32.664653 1 tasks_processing.go:74] worker 24 stopped. I0410 16:39:32.665047 1 tasks_processing.go:74] worker 10 stopped. I0410 16:39:32.665275 1 recorder.go:75] Recording config/featuregate with fingerprint=cbc1ce301d556e04d4582b4a23a50ff3ab48bf221bf764bd955e6018f0f21fb5 I0410 16:39:32.665303 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 32.31241ms to process 1 records I0410 16:39:32.666239 1 tasks_processing.go:74] worker 35 stopped. I0410 16:39:32.666910 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=4923ee2595b6e5c9ec2be87c07a69f8a270db5b6b789f801297afe4307c4a17a I0410 16:39:32.667060 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=1c70a563d99b8a7eaa8a10804fe873eb31288afb6eebcd90d5ca45c9537d5ea3 I0410 16:39:32.667756 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=8892cdb92853ed055e014dba9399da1916dd5a36bd3b1ee3e51ba6f842fa8b6d I0410 16:39:32.667779 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 39.102794ms to process 3 records I0410 16:39:32.667866 1 tasks_processing.go:74] worker 19 stopped. I0410 16:39:32.667984 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=726b03e996b056ba96ec5410419e2195183f71baf2f6458128db91f99d54466a I0410 16:39:32.668103 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=73d0ec867a13b7e429d8acf7220e2a08af5a9880eab314a3070bed43e28509a2 I0410 16:39:32.668115 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 37.93321ms to process 2 records I0410 16:39:32.668121 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 39.778281ms to process 0 records I0410 16:39:32.668148 1 tasks_processing.go:74] worker 53 stopped. I0410 16:39:32.668201 1 tasks_processing.go:74] worker 54 stopped. I0410 16:39:32.668248 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=fdee43660a5b942a5f98ebb3e5378a15114d9c266f26ce313a191eb828aeb2c8 I0410 16:39:32.668375 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=d61be9c1861270b63740359018e95bc73affa42a6001fd7390c6af736251f5a4 I0410 16:39:32.668447 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=ff867dac6a31649e0dcecc19e678c62e56d6fb150bb6754d8591febeb9d411d5 I0410 16:39:32.668491 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=3c90dce9692a06c150562a2522191a1af46b41a7a724941a468f2c9043d008c8 I0410 16:39:32.668531 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=6e89a97b7137e23d7ea3d5fa3b6717c8dc2ef3516e4f4a27349f3b423403fb7a I0410 16:39:32.668570 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=6a6a6324c0592617e79b43bbf8b0045d87584819731142dcfc4c8d47095ebbff I0410 16:39:32.668607 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=05fc36098e1414b85304b041e0521892e3f59a31a7a6017fe195faaf547735a1 I0410 16:39:32.668608 1 controller.go:212] Source scaController *sca.Controller is not ready I0410 16:39:32.668618 1 controller.go:212] Source clusterTransferController *clustertransfer.Controller is not ready I0410 16:39:32.668624 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0410 16:39:32.668629 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0410 16:39:32.668633 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0410 16:39:32.668656 1 controller.go:489] The operator is still being initialized I0410 16:39:32.668662 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=de8f26569fabaddf733a02b6e9c4e8b8c30f9c4a1e6113726c7e4fa59e04adfd I0410 16:39:32.668670 1 controller.go:512] The operator is healthy I0410 16:39:32.668699 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=80fdebc93159f53ecaee60f10db198eca837db22021ea18b5f44fe2aa2c3dc31 I0410 16:39:32.668727 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=4c835b43231ecea35b3192215755621093581d8a3e053524ba1efb936f430e6a I0410 16:39:32.668753 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=bd4e1996aac01d9bfeae04afefef98718b1f3d6e6aac9f1a02cb20eb62a91859 I0410 16:39:32.668761 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 40.388446ms to process 11 records I0410 16:39:32.668845 1 tasks_processing.go:74] worker 30 stopped. I0410 16:39:32.669012 1 recorder.go:75] Recording config/version with fingerprint=d50fc3b2e11dd4f4c0ec4e44193d7c458943beadce37fe1a1bbce82356fb046b I0410 16:39:32.669020 1 recorder.go:75] Recording config/id with fingerprint=40dfbef5743e71c41db15878d966ce420a4b7744e5593fc42359a8cb1fadbdca I0410 16:39:32.669025 1 gather.go:177] gatherer "clusterconfig" function "version" took 40.705049ms to process 2 records I0410 16:39:32.669035 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 37.809259ms to process 0 records I0410 16:39:32.669041 1 tasks_processing.go:74] worker 62 stopped. I0410 16:39:32.674928 1 tasks_processing.go:74] worker 46 stopped. I0410 16:39:32.675028 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0410 16:39:32.675045 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 45.311974ms to process 1 records I0410 16:39:32.677959 1 tasks_processing.go:74] worker 55 stopped. I0410 16:39:32.678632 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=ffdcfff955b881b77b60bcef4d3faecc9cbee95a219472339f6e5ef7c51e2723 I0410 16:39:32.678878 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=cb7ac99778a29b2d0ebd51d55ab259f05bc5d02d37c27cb05a1060bd72f54690 I0410 16:39:32.678889 1 gather.go:177] gatherer "clusterconfig" function "crds" took 50.805447ms to process 2 records I0410 16:39:32.679610 1 tasks_processing.go:74] worker 56 stopped. I0410 16:39:32.681243 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-575cd97545-lcd4l with fingerprint=eb87a182401dc49a6c294f404957efc6d3054d992a8fb637dc4cd070a284410e I0410 16:39:32.681514 1 recorder.go:75] Recording config/running_containers with fingerprint=33d5084e64ea8794976076e8c4ee6a3a93ebed28b999428049f0149497da0417 I0410 16:39:32.681561 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 52.473576ms to process 2 records I0410 16:39:32.683128 1 tasks_processing.go:74] worker 12 stopped. I0410 16:39:32.683146 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 55.937689ms to process 0 records I0410 16:39:32.683493 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0410 16:39:32.689084 1 tasks_processing.go:74] worker 41 stopped. E0410 16:39:32.689096 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0410 16:39:32.689102 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0410 16:39:32.689106 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0410 16:39:32.689115 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=26b6661162b099a0f5a279859b4f46c867929a79d9a4a41fde4be4e6fe138018 I0410 16:39:32.689138 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0410 16:39:32.689145 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0410 16:39:32.689149 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=c93090eb0d2a4736885abeb79c91680cfd01fda46464f83456b085d4dc8239f0 I0410 16:39:32.689167 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0410 16:39:32.689205 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0410 16:39:32.689213 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0410 16:39:32.689218 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 61.440369ms to process 7 records I0410 16:39:32.695446 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0410 16:39:32.695458 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0410 16:39:32.695477 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0410 16:39:32.695984 1 tasks_processing.go:74] worker 21 stopped. E0410 16:39:32.696001 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0410 16:39:32.696009 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2pjda9copft31brcj754brber09es9ns-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2pjda9copft31brcj754brber09es9ns-primary-cert-bundle-secret" not found I0410 16:39:32.696087 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=3dab80850d98200402df6122b04374efe46756ab7a60fc111f37662b632b58cc I0410 16:39:32.696103 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 62.487515ms to process 1 records I0410 16:39:32.701905 1 base_controller.go:82] Caches are synced for ConfigController I0410 16:39:32.701915 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0410 16:39:32.706465 1 configmapobserver.go:84] configmaps "insights-config" not found I0410 16:39:32.735060 1 base_controller.go:82] Caches are synced for LoggingSyncer I0410 16:39:32.735073 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0410 16:39:32.735671 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-mj4fr pod in namespace openshift-ingress-canary (previous: false). I0410 16:39:32.766505 1 cluster_transfer.go:102] no available accepted cluster transfer I0410 16:39:32.766520 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0410 16:39:32.840809 1 sca.go:266] etc-pki-entitlement secret successfully created I0410 16:39:32.840826 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated I0410 16:39:32.881005 1 tasks_processing.go:74] worker 52 stopped. I0410 16:39:32.881112 1 recorder.go:75] Recording events/openshift-ingress-operator with fingerprint=9226f7a05930ffbd415111648c0aca126e6455bb4e9dc9e6ff2f4ba96a2517ef I0410 16:39:32.881190 1 recorder.go:75] Recording events/openshift-ingress with fingerprint=57af4f00dd978fb72d44792be80bfcbd13e33dd9b416114f67677e3d5a532f74 I0410 16:39:32.881210 1 recorder.go:75] Recording events/openshift-ingress-canary with fingerprint=fa3594aaed45c2325247b71afcfe631ba83d807f88c161e6a85eb749d4483c03 I0410 16:39:32.881230 1 recorder.go:75] Recording config/pod/openshift-ingress/logs/router-default-86b8698cdb-jn4jq/router_current.log with fingerprint=04c88c9a9cabdfa877af20ffe33457bb21bc69b2addc6041b6a5d82061f038ba I0410 16:39:32.881268 1 recorder.go:75] Recording config/pod/openshift-ingress/logs/router-default-b7c798989-r74cd/router_current.log with fingerprint=86af1acc2581e1cbdd86aa178cdc99864f8310a7444e6248032b2a1116caa2ba I0410 16:39:32.881275 1 recorder.go:75] Recording config/pod/openshift-ingress-canary/logs/ingress-canary-mj4fr/serve-healthcheck-canary_current.log with fingerprint=a829167bacad6f38c2ab6798a421b71556a30560964a9171db70583b303d5b2a I0410 16:39:32.881282 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 251.202048ms to process 6 records W0410 16:39:33.646844 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0410 16:39:34.646870 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0410 16:39:35.093524 1 tasks_processing.go:74] worker 2 stopped. I0410 16:39:35.093673 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=7165e3b23088321a8861ad945720f8c2084f0cbcfd2a7a9344b3d70cfd6c6e13 I0410 16:39:35.093733 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=b1d3a57201f472fba248e6e22732d03f4ebac77f3d7204455295e9c6f31f376a I0410 16:39:35.093773 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=6022cbf432629eaef5a0fc103788b8cf614777965f31672d42443ff39e2708d6 I0410 16:39:35.093788 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0410 16:39:35.093822 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=9482bd60153e29dc7ac77dae6134cbb5342e2430502f6f46596697ee6a68f126 I0410 16:39:35.093848 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0410 16:39:35.093880 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=059775eadcc00b74ef120e584d771e8e29c8a88014c04ae13811e1ed40307d7f I0410 16:39:35.093921 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=62ee5fc050b3b003aa99f92904545dd9f6106c9e4e74759d93d2a3858b094e86 I0410 16:39:35.093954 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=8b1ca5b917efefa23a576ce77e0a329ee153ed43cb1c01d6b1d964b58047c577 I0410 16:39:35.093995 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=fadbdfd9d908398af1ca2d3c90c2901a968d600b94fcacec5b0abd7543f26808 I0410 16:39:35.094272 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=e276516385eed25d8508cb4fa3087d47a13efbda79025caa9a2071f8a8aa4ba9 I0410 16:39:35.094304 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=19db953efaead63eddc7b74a97f21b8f508298a1a8acbf41d63d1a9023ac2c43 I0410 16:39:35.094319 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0410 16:39:35.094350 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=159d7d234262987e4329396b393bbb79c8fee3e135b44e036dc944be3a81d06d I0410 16:39:35.094367 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0410 16:39:35.094400 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=8e93d26f41890a5ae6a549ee4eff51d70c18e28e77d5ab264a69d10b2e84d318 I0410 16:39:35.094435 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0410 16:39:35.094466 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=c2c427ee87e50e9a490b1c7a72cc6736779ed42bff489a6298ce52c828c13e38 I0410 16:39:35.094480 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0410 16:39:35.094509 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=4b0d6ff94d95455b6a0fe91a652a26535a713a82164480a5c32543a99c9a714b I0410 16:39:35.094620 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=bb0d98c3a17ff2d4756a29df458d76e450a721fb3cb0f8131dd4518df3ce247b I0410 16:39:35.094633 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0410 16:39:35.094661 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0410 16:39:35.094695 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0410 16:39:35.094726 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=35f49642fa9658107ade6e673bd118e79e3a644acfd402a28bf6c8fd89b64f26 I0410 16:39:35.094762 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=a7c67d40835eb34b27901bd89d18f0668e81937f8d2bf1ce3235b95a52da5e87 I0410 16:39:35.094776 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0410 16:39:35.094802 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=ce4e65e5b91bb47d1bec93ab19d78f88884b5fb8153b2ac5beca12febc57bc4e I0410 16:39:35.094817 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0410 16:39:35.094844 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=390aaf502486ec03d7e1f8c6a60422d419d95dc8670848e2d5a06e519080b4ad I0410 16:39:35.094857 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0410 16:39:35.094880 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=56493daa3a42edaa27d6c5a3c7b3bdf1d27549a7ee035a03f6baf4b738f92f94 I0410 16:39:35.094906 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=0c8d19d525373fd20e450aa66622b3f7b1b761d65544fab72ca39d07ceb845c3 I0410 16:39:35.094931 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=dd0f843958f071dca40c6920fa5eeea88f23430b2e3c76afe10fa6e8a1368ada I0410 16:39:35.094963 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=35df12ba8114efb56ff10d2249c129c31ceede45479f7a5a6e8fcaf15bb278bc I0410 16:39:35.094977 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0410 16:39:35.095009 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=231848c1c97550ff0fb314c34c4c15635120c21c10493150952cc80c42c4ed62 I0410 16:39:35.095034 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0410 16:39:35.095048 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0410 16:39:35.095059 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.465846398s to process 39 records W0410 16:39:35.646875 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0410 16:39:36.646011 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0410 16:39:37.644779 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0410 16:39:37.644803 1 tasks_processing.go:74] worker 13 stopped. E0410 16:39:37.644814 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0410 16:39:37.644824 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0410 16:39:37.644835 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.017673888s to process 1 records I0410 16:39:45.474013 1 tasks_processing.go:74] worker 37 stopped. I0410 16:39:45.474048 1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b I0410 16:39:45.474060 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.841038158s to process 1 records I0410 16:39:46.237713 1 tasks_processing.go:74] worker 29 stopped. I0410 16:39:46.237985 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=3f42842a95abf42f7e7c2b33333f34e223bd0ca1641d8e40233cb263a6e89e93 I0410 16:39:46.238002 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.61057654s to process 1 records E0410 16:39:46.238065 1 periodic.go:247] "Unhandled Error" err="clusterconfig failed after 13.611s with: function \"active_alerts\" failed with an error, function \"silenced_alerts\" failed with an error, function \"metrics\" failed with an error, function \"tsdb_status\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machines\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"support_secret\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0410 16:39:46.239196 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "active_alerts" failed with an error, function "silenced_alerts" failed with an error, function "metrics" failed with an error, function "tsdb_status" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machines" failed with an error, function "machine_healthchecks" failed with an error, function "support_secret" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0410 16:39:46.239215 1 periodic.go:209] Running workloads gatherer I0410 16:39:46.239235 1 tasks_processing.go:45] number of workers: 2 I0410 16:39:46.239246 1 tasks_processing.go:69] worker 1 listening for tasks. I0410 16:39:46.239252 1 tasks_processing.go:71] worker 1 working on workload_info task. I0410 16:39:46.239260 1 tasks_processing.go:69] worker 0 listening for tasks. I0410 16:39:46.239275 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0410 16:39:46.246811 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-37.ec2.internal... I0410 16:39:46.268059 1 tasks_processing.go:74] worker 0 stopped. I0410 16:39:46.268076 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 28.768902ms to process 0 records I0410 16:39:51.945176 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 6s I0410 16:39:51.959384 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 20s for image data I0410 16:39:51.968570 1 gather_workloads_info.go:387] No image sha256:a498046d64605bcccee2440aa4f04a4602baaae263cf01d977ec5208e876b1fd (10ms) I0410 16:39:51.977479 1 gather_workloads_info.go:387] No image sha256:765f0d23b637f685f98a31bd47c131b03cf72a40761a3f9a9d6320faa3c33733 (9ms) I0410 16:39:51.985998 1 gather_workloads_info.go:387] No image sha256:a258c226562adb14e3a163a1940938526ee6a0928982a7667d85d9a7334ce639 (9ms) I0410 16:39:51.993873 1 gather_workloads_info.go:387] No image sha256:5a95c19d82767e0235b4edb4a0536482c816904897aae1dc3eb255cb52b87a9f (8ms) I0410 16:39:52.004820 1 gather_workloads_info.go:387] No image sha256:a0105d1eb62cf6ac9e5e2ef28d3e89bf6dc514bc594fc7090fe5a5ee18a09c87 (11ms) I0410 16:39:52.012854 1 gather_workloads_info.go:387] No image sha256:2904a78e2eb73fd6a9bb94c105c2a056831fb4113fbb7b0607c50adc9d879c9b (8ms) I0410 16:39:52.020904 1 gather_workloads_info.go:387] No image sha256:653c666f842c13e0baae2e89a9b1efe0e2ef56f621ffb5b32005115d2a26ab8c (8ms) I0410 16:39:52.029746 1 gather_workloads_info.go:398] Found image sha256:32d4561d4bf1b3c4e3a7878f918f9c1422bab85bdf8711fbdf905d59692fee81 (9ms) I0410 16:39:52.038763 1 gather_workloads_info.go:387] No image sha256:521712486e2c6e3c020dad6a1cb340db8e55665b69f7c208fab9cd9e965fd588 (9ms) I0410 16:39:52.047449 1 gather_workloads_info.go:398] Found image sha256:b3909bf664c77097f75b3768830863d642eed3815dab2bfb4415c771ca2d5007 (9ms) I0410 16:39:52.068996 1 gather_workloads_info.go:387] No image sha256:47154813651033d59751fb655a384dbffb64dd26f10bd7f3be0c3128d0486356 (21ms) I0410 16:39:52.169149 1 gather_workloads_info.go:387] No image sha256:943018739e3db1763c3184b460dbc409e058abbac76d57b9927faad317be85e4 (100ms) I0410 16:39:52.269159 1 gather_workloads_info.go:387] No image sha256:695cf2f0cc07683c2a3ce1eaf3e56fe18abc6e2bac716f7d9843f5d173b9df52 (100ms) I0410 16:39:52.369162 1 gather_workloads_info.go:387] No image sha256:50197f22710766515f67944a779e00dd9ae3d17b18732d7324a970353b11f292 (100ms) I0410 16:39:52.468729 1 gather_workloads_info.go:387] No image sha256:56a85660a445eced5c79a595a0eccf590087c5672d50f49d4c25ad52f9a44f04 (100ms) I0410 16:39:52.570040 1 gather_workloads_info.go:387] No image sha256:5f0b67cfbbc381243fb91ccc17345b56d05f4d717c667e8c644e5bf05633ba71 (101ms) I0410 16:39:52.669273 1 gather_workloads_info.go:387] No image sha256:a56211d075aa43cbb491f669a5b2e46ee023dc95b7d51dbac28f463948c5ad61 (99ms) I0410 16:39:52.769063 1 gather_workloads_info.go:387] No image sha256:25bda9d34e23ba6bc9df5cf7104bdb237269740a3a76ee025f88b34e68b7e2b5 (100ms) I0410 16:39:52.877701 1 gather_workloads_info.go:387] No image sha256:4556896f77307821531ef91b7b7faccb82b824ea695693b2989f597f0deca038 (109ms) I0410 16:39:52.972725 1 gather_workloads_info.go:387] No image sha256:91828234f107c068c8a4966d08370ae7b73e637651dbc6d92c18c4553402c22c (95ms) I0410 16:39:53.069269 1 gather_workloads_info.go:387] No image sha256:76cc47f3775560477ec548c7b6fddbd99112ad6604f790697eef546576051fb7 (97ms) I0410 16:39:53.173527 1 gather_workloads_info.go:387] No image sha256:3612c5a433effd17b7aff7ff6edf4fb27b61153908e3a0294158e4886a1d600d (104ms) I0410 16:39:53.269436 1 gather_workloads_info.go:387] No image sha256:03cf4cd7ef1518610c6c7b3ad27d1622d82e98e3dc6e3f8e5d0fceb5c8d3786e (96ms) I0410 16:39:53.369572 1 gather_workloads_info.go:387] No image sha256:3d6ea9e1633f38070718a1357fcb80e6cb3932aafcc4d65e659795b186c5da15 (100ms) I0410 16:39:53.468828 1 gather_workloads_info.go:387] No image sha256:289816958633a763a72dbc44e1dad40466223164e7e253039514f0d974ea5d21 (99ms) I0410 16:39:53.570163 1 gather_workloads_info.go:387] No image sha256:c940ea87e7d133d75ba0002ef00c0806825eed3db8094cdb260d1bac18127733 (101ms) I0410 16:39:53.670175 1 gather_workloads_info.go:387] No image sha256:1a2532940843248c57d52141185dd71fbc393ab28b65d48f682038632c1dbbad (100ms) I0410 16:39:53.769157 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (99ms) I0410 16:39:53.869660 1 gather_workloads_info.go:387] No image sha256:7adc1eab05d6724c76ba751f6df816b08d6e70b78dee9eb94fa6fd9690542c98 (100ms) I0410 16:39:53.979896 1 gather_workloads_info.go:387] No image sha256:2598489729a4b258e4ecda4a06f6875133f2a10ced5c5241f8a57a8a05418e36 (110ms) I0410 16:39:54.079675 1 gather_workloads_info.go:387] No image sha256:36b9e89c3cfcf1ab9ae500486e38afb6862cba48cb0b4d84a09508ab8f3d299f (100ms) I0410 16:39:54.168705 1 gather_workloads_info.go:387] No image sha256:7b31223098f08328f5ddea8e5b871dbbd5f5a61ec550e8956f66793c0c6031a9 (89ms) I0410 16:39:54.269425 1 gather_workloads_info.go:387] No image sha256:2e564f336c77116053f34d4201d364d8da04e789cfffa0ea422574c95f2d6404 (101ms) I0410 16:39:54.369264 1 gather_workloads_info.go:387] No image sha256:0a99240166165eb5718e7516a43282fe32df9c7c5e809b31b58abe44e42ff94d (100ms) I0410 16:39:54.468811 1 gather_workloads_info.go:387] No image sha256:2e57e192c3c1240fd935dcd55c8fde5e70e78bf81d6176c96edf21fafe59f8ba (100ms) I0410 16:39:54.468841 1 tasks_processing.go:74] worker 1 stopped. I0410 16:39:54.469111 1 recorder.go:75] Recording config/workload_info with fingerprint=a0b1f95d406138b31055edffe8eb57b8fc98d961fc7fa1c56f95334c8922684c I0410 16:39:54.469125 1 gather.go:177] gatherer "workloads" function "workload_info" took 8.22958177s to process 1 records I0410 16:39:54.469138 1 periodic.go:256] Periodic gather workloads completed in 8.229s I0410 16:39:54.469145 1 controllerstatus.go:80] name=periodic-workloads healthy=true reason= message= I0410 16:39:54.469149 1 periodic.go:209] Running conditional gatherer I0410 16:39:54.475007 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules I0410 16:39:54.937561 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer E0410 16:39:54.940343 1 conditional_gatherer.go:324] unable to update alerts cache: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?match%5B%5D=ALERTS%7Balertstate%3D%22firing%22%7D&query=ALERTS": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0410 16:39:54.940427 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0410 16:39:54.946348 1 conditional_gatherer.go:392] cluster version is '4.20.8' E0410 16:39:54.946368 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0410 16:39:54.946375 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0410 16:39:54.946381 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0410 16:39:54.946385 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0410 16:39:54.946396 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0410 16:39:54.946400 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0410 16:39:54.946420 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0410 16:39:54.946425 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0410 16:39:54.946429 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0410 16:39:54.946434 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0410 16:39:54.946454 1 tasks_processing.go:45] number of workers: 3 I0410 16:39:54.946469 1 tasks_processing.go:69] worker 2 listening for tasks. I0410 16:39:54.946478 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0410 16:39:54.946478 1 tasks_processing.go:69] worker 0 listening for tasks. I0410 16:39:54.946489 1 tasks_processing.go:69] worker 1 listening for tasks. I0410 16:39:54.946494 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0410 16:39:54.946497 1 tasks_processing.go:74] worker 1 stopped. I0410 16:39:54.946505 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0410 16:39:54.946574 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=9fafdcbf373f1f367f1d69551ea59114592cce1681189be0f3dde774e0530661 I0410 16:39:54.946591 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.389µs to process 1 records I0410 16:39:54.946631 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0410 16:39:54.946643 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.365µs to process 1 records I0410 16:39:54.946652 1 tasks_processing.go:74] worker 0 stopped. I0410 16:39:54.946733 1 tasks_processing.go:74] worker 2 stopped. I0410 16:39:54.946745 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 215.742µs to process 0 records I0410 16:39:54.946768 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0410 16:39:54.946785 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0410 16:39:54.971723 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=d93fd1fdb437aa4749f4f662484d2f0448923008d470dd82ab32a2eba35ca2ce I0410 16:39:54.971880 1 diskrecorder.go:70] Writing 105 records to /var/lib/insights-operator/insights-2026-04-10-163954.tar.gz I0410 16:39:54.978981 1 diskrecorder.go:51] Wrote 105 records to disk in 7ms I0410 16:39:54.979012 1 periodic.go:278] Gathering cluster info every 2h0m0s I0410 16:39:54.979027 1 periodic.go:279] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0410 16:41:02.627981 1 diskrecorder.go:223] Found files to send: insights-2026-04-10-163954.tar.gz I0410 16:41:02.628012 1 insightsuploader.go:150] Checking archives to upload periodically every 16m26.361384989s I0410 16:41:02.628021 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0410 16:41:02.637810 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0410 16:41:02.831092 1 requests.go:87] Successfully reported id=2026-04-10T16:41:02Z x-rh-insights-request-id=8c5c4ce301764d37bdc634eb464b9403, wrote=48857 I0410 16:41:02.831157 1 insightsuploader.go:187] Uploaded report successfully in 203.128284ms I0410 16:41:02.831183 1 controller.go:128] Initializing last reported time to 2026-04-10T16:41:02Z I0410 16:41:02.831203 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0410 16:41:02.831212 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0410 16:41:02.831220 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0410 16:41:02.836876 1 controller.go:512] The operator is healthy I0410 16:41:32.641564 1 controller.go:512] The operator is healthy I0410 16:42:08.575776 1 insightsreport.go:137] Pulling report from smart-proxy I0410 16:42:08.575806 1 insightsreport.go:149] Retrieving report I0410 16:42:08.582967 1 requests.go:110] Retrieving report for cluster: 59f18ac5-46e1-44b9-b3e4-a5a00e24d247 I0410 16:42:08.582980 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/59f18ac5-46e1-44b9-b3e4-a5a00e24d247/reports I0410 16:42:08.585655 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/59f18ac5-46e1-44b9-b3e4-a5a00e24d247/reports I0410 16:42:08.864023 1 insightsreport.go:184] Report retrieved I0410 16:42:08.889749 1 insightsreport.go:239] Report retrieved correctly I0410 16:43:32.641315 1 controller.go:512] The operator is healthy I0410 16:44:32.620571 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0410 16:44:32.628720 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0410 16:44:32.628739 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0410 16:44:32.633081 1 secretconfigobserver.go:119] support secret does not exist I0410 16:45:32.642290 1 controller.go:512] The operator is healthy I0410 16:47:32.644686 1 controller.go:512] The operator is healthy I0410 16:49:32.618669 1 configmapobserver.go:84] configmaps "insights-config" not found I0410 16:49:32.623508 1 configmapobserver.go:84] configmaps "insights-config" not found I0410 16:49:32.633600 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0410 16:49:32.638592 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0410 16:49:32.638607 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0410 16:49:32.640521 1 controller.go:512] The operator is healthy I0410 16:49:32.642990 1 secretconfigobserver.go:119] support secret does not exist I0410 16:51:32.645684 1 controller.go:512] The operator is healthy I0410 16:53:32.642381 1 controller.go:512] The operator is healthy I0410 16:54:32.643286 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0410 16:54:32.649474 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0410 16:54:32.649497 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0410 16:54:32.653822 1 secretconfigobserver.go:119] support secret does not exist I0410 16:55:32.641085 1 controller.go:512] The operator is healthy I0410 16:57:32.641650 1 controller.go:512] The operator is healthy I0410 16:57:34.194120 1 insightsuploader.go:146] Nothing to report since 2026-04-10T16:41:02Z I0410 16:59:32.616524 1 configmapobserver.go:84] configmaps "insights-config" not found I0410 16:59:32.640821 1 controller.go:512] The operator is healthy I0410 16:59:32.654663 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0410 16:59:32.659567 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0410 16:59:32.659583 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0410 16:59:32.663860 1 secretconfigobserver.go:119] support secret does not exist I0410 17:01:32.643955 1 controller.go:512] The operator is healthy I0410 17:03:32.641203 1 controller.go:512] The operator is healthy I0410 17:04:32.664677 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0410 17:04:32.673163 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0410 17:04:32.673181 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0410 17:04:32.677516 1 secretconfigobserver.go:119] support secret does not exist I0410 17:05:32.643603 1 controller.go:512] The operator is healthy I0410 17:07:32.643614 1 controller.go:512] The operator is healthy I0410 17:09:32.619735 1 configmapobserver.go:84] configmaps "insights-config" not found I0410 17:09:32.640652 1 controller.go:512] The operator is healthy I0410 17:09:32.677711 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0410 17:09:32.682922 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0410 17:09:32.682938 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0410 17:09:32.687101 1 secretconfigobserver.go:119] support secret does not exist