I0504 20:40:26.578895 1 cmd.go:253] Using service-serving-cert provided certificates I0504 20:40:26.579357 1 observer_polling.go:159] Starting file observer I0504 20:40:27.059123 1 operator.go:76] Starting insights-operator v0.0.0-master+$Format:%H$ I0504 20:40:27.059358 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0504 20:40:27.060529 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0504 20:40:27.061287 1 secure_serving.go:57] Forcing use of http/1.1 only W0504 20:40:27.061309 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0504 20:40:27.061318 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0504 20:40:27.061323 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0504 20:40:27.061329 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0504 20:40:27.061332 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0504 20:40:27.061335 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0504 20:40:27.064928 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"58250211-bdd5-458f-9fb2-d43d3a104b79", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "HyperShiftOnlyDynamicResourceAllocation", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryption", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLM", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "OpenShiftPodSecurityAdmission", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}} I0504 20:40:27.064919 1 operator.go:141] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNSInstall GatewayAPI GatewayAPIController HighlyAvailableArbiter HyperShiftOnlyDynamicResourceAllocation ImageStreamImportMode ImageVolume KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS ManagedBootImagesAzure ManagedBootImagesvSphere MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages PreconfiguredUDNAddresses ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SigstoreImageVerification SigstoreImageVerificationPKI StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSDualStackInstall AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureDualStackInstall AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement CBORServingAndStorage CRDCompatibilityRequirementOperator ClientsAllowCBOR ClientsPreferCBOR ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterAPIMachineManagement ClusterAPIMachineManagementVSphere ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud EtcdBackendQuota EventTTL EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall GCPDualStackInstall ImageModeStatusReporting IngressControllerDynamicConfigurationManager InsightsConfig InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryption KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesCPMS MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutableCSINodeAllocatableCount MutatingAdmissionPolicy NewOLM NewOLMBoxCutterRuntime NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterInstall NutanixMultiSubnets OSStreams OVNObservability OnPremDNSRecords OpenShiftPodSecurityAdmission ProvisioningRequestAvailable SELinuxMount ShortCertRotation SignatureStores TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeGroupSnapshot] I0504 20:40:27.067071 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0504 20:40:27.067111 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0504 20:40:27.067113 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0504 20:40:27.067112 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0504 20:40:27.067129 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0504 20:40:27.067131 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0504 20:40:27.067420 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0504 20:40:27.067654 1 secure_serving.go:213] Serving securely on [::]:8443 I0504 20:40:27.067681 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0504 20:40:27.070490 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0504 20:40:27.070516 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0504 20:40:27.070651 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0504 20:40:27.075763 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 20:40:27.075781 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0504 20:40:27.080115 1 secretconfigobserver.go:119] support secret does not exist I0504 20:40:27.085401 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 20:40:27.090274 1 secretconfigobserver.go:119] support secret does not exist I0504 20:40:27.092548 1 recorder.go:176] Pruning old reports every 7h31m19s, max age is 288h0m0s I0504 20:40:27.097809 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0504 20:40:27.097832 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0504 20:40:27.097811 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0504 20:40:27.097865 1 insightsreport.go:296] Starting report retriever I0504 20:40:27.097864 1 periodic.go:216] Running clusterconfig gatherer I0504 20:40:27.097871 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0504 20:40:27.097908 1 tasks_processing.go:45] number of workers: 64 I0504 20:40:27.097935 1 tasks_processing.go:69] worker 7 listening for tasks. I0504 20:40:27.097948 1 tasks_processing.go:69] worker 4 listening for tasks. I0504 20:40:27.097953 1 tasks_processing.go:69] worker 5 listening for tasks. I0504 20:40:27.097952 1 tasks_processing.go:69] worker 3 listening for tasks. I0504 20:40:27.097959 1 tasks_processing.go:71] worker 4 working on aggregated_monitoring_cr_names task. I0504 20:40:27.097964 1 tasks_processing.go:71] worker 3 working on feature_gates task. I0504 20:40:27.097966 1 tasks_processing.go:69] worker 8 listening for tasks. I0504 20:40:27.097969 1 tasks_processing.go:69] worker 6 listening for tasks. I0504 20:40:27.097970 1 tasks_processing.go:69] worker 0 listening for tasks. I0504 20:40:27.097959 1 tasks_processing.go:71] worker 5 working on image_pruners task. I0504 20:40:27.097982 1 tasks_processing.go:69] worker 19 listening for tasks. I0504 20:40:27.097983 1 tasks_processing.go:69] worker 20 listening for tasks. I0504 20:40:27.097984 1 tasks_processing.go:69] worker 10 listening for tasks. I0504 20:40:27.097989 1 tasks_processing.go:69] worker 11 listening for tasks. I0504 20:40:27.097991 1 tasks_processing.go:69] worker 15 listening for tasks. I0504 20:40:27.097980 1 tasks_processing.go:69] worker 14 listening for tasks. I0504 20:40:27.097998 1 tasks_processing.go:69] worker 21 listening for tasks. I0504 20:40:27.098000 1 tasks_processing.go:69] worker 13 listening for tasks. I0504 20:40:27.098002 1 tasks_processing.go:71] worker 14 working on authentication task. I0504 20:40:27.098006 1 tasks_processing.go:69] worker 25 listening for tasks. I0504 20:40:27.098007 1 tasks_processing.go:69] worker 22 listening for tasks. I0504 20:40:27.098013 1 tasks_processing.go:69] worker 23 listening for tasks. I0504 20:40:27.098017 1 tasks_processing.go:69] worker 26 listening for tasks. I0504 20:40:27.098010 1 tasks_processing.go:71] worker 15 working on networks task. I0504 20:40:27.098022 1 tasks_processing.go:69] worker 24 listening for tasks. I0504 20:40:27.098021 1 tasks_processing.go:71] worker 8 working on monitoring_persistent_volumes task. I0504 20:40:27.098030 1 tasks_processing.go:71] worker 10 working on pdbs task. I0504 20:40:27.098043 1 tasks_processing.go:69] worker 16 listening for tasks. I0504 20:40:27.098040 1 tasks_processing.go:71] worker 19 working on install_plans task. I0504 20:40:27.098051 1 tasks_processing.go:71] worker 0 working on active_alerts task. I0504 20:40:27.098051 1 tasks_processing.go:71] worker 6 working on jaegers task. I0504 20:40:27.098072 1 tasks_processing.go:69] worker 27 listening for tasks. I0504 20:40:27.098113 1 tasks_processing.go:69] worker 47 listening for tasks. I0504 20:40:27.098113 1 tasks_processing.go:69] worker 28 listening for tasks. I0504 20:40:27.098126 1 tasks_processing.go:69] worker 30 listening for tasks. I0504 20:40:27.098134 1 tasks_processing.go:69] worker 31 listening for tasks. I0504 20:40:27.098134 1 tasks_processing.go:69] worker 56 listening for tasks. I0504 20:40:27.098142 1 tasks_processing.go:69] worker 48 listening for tasks. I0504 20:40:27.098134 1 tasks_processing.go:69] worker 2 listening for tasks. I0504 20:40:27.098152 1 tasks_processing.go:69] worker 49 listening for tasks. I0504 20:40:27.098147 1 tasks_processing.go:69] worker 60 listening for tasks. I0504 20:40:27.098155 1 tasks_processing.go:69] worker 41 listening for tasks. I0504 20:40:27.098163 1 tasks_processing.go:69] worker 50 listening for tasks. I0504 20:40:27.098147 1 tasks_processing.go:71] worker 11 working on image_registries task. I0504 20:40:27.098169 1 tasks_processing.go:69] worker 35 listening for tasks. I0504 20:40:27.098173 1 tasks_processing.go:69] worker 51 listening for tasks. I0504 20:40:27.098171 1 tasks_processing.go:69] worker 59 listening for tasks. I0504 20:40:27.097959 1 tasks_processing.go:69] worker 29 listening for tasks. I0504 20:40:27.098181 1 tasks_processing.go:69] worker 61 listening for tasks. I0504 20:40:27.098184 1 tasks_processing.go:69] worker 36 listening for tasks. I0504 20:40:27.098164 1 tasks_processing.go:69] worker 58 listening for tasks. I0504 20:40:27.098186 1 tasks_processing.go:69] worker 33 listening for tasks. I0504 20:40:27.098193 1 tasks_processing.go:69] worker 62 listening for tasks. I0504 20:40:27.098195 1 tasks_processing.go:69] worker 37 listening for tasks. I0504 20:40:27.098190 1 tasks_processing.go:69] worker 34 listening for tasks. I0504 20:40:27.098197 1 tasks_processing.go:69] worker 55 listening for tasks. I0504 20:40:27.098200 1 tasks_processing.go:69] worker 38 listening for tasks. I0504 20:40:27.098197 1 tasks_processing.go:69] worker 54 listening for tasks. I0504 20:40:27.098201 1 tasks_processing.go:69] worker 18 listening for tasks. I0504 20:40:27.098198 1 tasks_processing.go:69] worker 46 listening for tasks. I0504 20:40:27.098208 1 tasks_processing.go:71] worker 22 working on sap_pods task. I0504 20:40:27.098207 1 tasks_processing.go:71] worker 7 working on ingress task. I0504 20:40:27.098215 1 tasks_processing.go:71] worker 47 working on olm_operators task. I0504 20:40:27.098213 1 tasks_processing.go:71] worker 16 working on metrics task. I0504 20:40:27.098217 1 tasks_processing.go:71] worker 24 working on container_images task. I0504 20:40:27.098215 1 tasks_processing.go:71] worker 27 working on config_maps task. I0504 20:40:27.098239 1 tasks_processing.go:71] worker 46 working on clusterroles task. I0504 20:40:27.098285 1 tasks_processing.go:71] worker 28 working on sap_datahubs task. I0504 20:40:27.098029 1 tasks_processing.go:69] worker 17 listening for tasks. I0504 20:40:27.098026 1 tasks_processing.go:71] worker 20 working on openshift_machine_api_events task. I0504 20:40:27.098302 1 tasks_processing.go:71] worker 17 working on qemu_kubevirt_launcher_logs task. I0504 20:40:27.098303 1 tasks_processing.go:71] worker 30 working on cluster_apiserver task. I0504 20:40:27.098155 1 tasks_processing.go:69] worker 57 listening for tasks. I0504 20:40:27.098159 1 tasks_processing.go:69] worker 39 listening for tasks. I0504 20:40:27.098175 1 tasks_processing.go:69] worker 43 listening for tasks. I0504 20:40:27.098180 1 tasks_processing.go:69] worker 32 listening for tasks. I0504 20:40:27.098419 1 tasks_processing.go:71] worker 56 working on oauths task. I0504 20:40:27.098424 1 tasks_processing.go:71] worker 62 working on service_accounts task. I0504 20:40:27.098441 1 tasks_processing.go:71] worker 60 working on openstack_dataplanedeployments task. I0504 20:40:27.098480 1 tasks_processing.go:71] worker 38 working on operators_pods_and_events task. I0504 20:40:27.098538 1 tasks_processing.go:71] worker 26 working on nodenetworkstates task. I0504 20:40:27.098558 1 tasks_processing.go:71] worker 54 working on node_logs task. I0504 20:40:27.098565 1 tasks_processing.go:71] worker 48 working on image task. I0504 20:40:27.098646 1 tasks_processing.go:71] worker 41 working on storage_cluster task. I0504 20:40:27.097965 1 tasks_processing.go:69] worker 1 listening for tasks. I0504 20:40:27.098183 1 tasks_processing.go:69] worker 44 listening for tasks. I0504 20:40:27.098188 1 tasks_processing.go:69] worker 53 listening for tasks. I0504 20:40:27.098669 1 tasks_processing.go:71] worker 44 working on certificate_signing_requests task. I0504 20:40:27.097976 1 tasks_processing.go:69] worker 9 listening for tasks. I0504 20:40:27.098678 1 tasks_processing.go:71] worker 9 working on schedulers task. I0504 20:40:27.098692 1 tasks_processing.go:71] worker 2 working on machine_sets task. I0504 20:40:27.098191 1 tasks_processing.go:69] worker 45 listening for tasks. I0504 20:40:27.098713 1 tasks_processing.go:71] worker 45 working on machine_healthchecks task. I0504 20:40:27.098779 1 tasks_processing.go:71] worker 18 working on ceph_cluster task. I0504 20:40:27.097997 1 tasks_processing.go:69] worker 12 listening for tasks. I0504 20:40:27.098896 1 tasks_processing.go:71] worker 39 working on machine_configs task. I0504 20:40:27.098670 1 tasks_processing.go:71] worker 53 working on sap_config task. I0504 20:40:27.098204 1 tasks_processing.go:71] worker 23 working on storage_classes task. I0504 20:40:27.098181 1 tasks_processing.go:69] worker 52 listening for tasks. I0504 20:40:27.098204 1 tasks_processing.go:69] worker 63 listening for tasks. I0504 20:40:27.098208 1 tasks_processing.go:71] worker 25 working on lokistack task. I0504 20:40:27.099039 1 tasks_processing.go:71] worker 63 working on version task. I0504 20:40:27.098209 1 tasks_processing.go:71] worker 21 working on machine_config_pools task. I0504 20:40:27.098205 1 tasks_processing.go:71] worker 13 working on openstack_version task. I0504 20:40:27.098298 1 tasks_processing.go:71] worker 59 working on validating_webhook_configurations task. I0504 20:40:27.098336 1 tasks_processing.go:71] worker 37 working on node_features task. I0504 20:40:27.098349 1 tasks_processing.go:71] worker 61 working on silenced_alerts task. I0504 20:40:27.098355 1 tasks_processing.go:71] worker 36 working on container_runtime_configs task. I0504 20:40:27.098343 1 tasks_processing.go:71] worker 29 working on openstack_controlplanes task. I0504 20:40:27.098359 1 tasks_processing.go:71] worker 58 working on tsdb_status task. I0504 20:40:27.098363 1 tasks_processing.go:71] worker 33 working on proxies task. I0504 20:40:27.098148 1 tasks_processing.go:69] worker 40 listening for tasks. I0504 20:40:27.099651 1 tasks_processing.go:71] worker 40 working on infrastructures task. I0504 20:40:27.098413 1 tasks_processing.go:71] worker 31 working on openshift_logging task. I0504 20:40:27.098408 1 tasks_processing.go:71] worker 49 working on mutating_webhook_configurations task. I0504 20:40:27.098169 1 tasks_processing.go:69] worker 42 listening for tasks. I0504 20:40:27.099785 1 tasks_processing.go:71] worker 42 working on crds task. I0504 20:40:27.098879 1 tasks_processing.go:71] worker 50 working on pod_network_connectivity_checks task. I0504 20:40:27.098889 1 tasks_processing.go:71] worker 43 working on overlapping_namespace_uids task. I0504 20:40:27.098893 1 tasks_processing.go:71] worker 57 working on machines task. I0504 20:40:27.098905 1 tasks_processing.go:71] worker 32 working on machine_autoscalers task. I0504 20:40:27.098912 1 tasks_processing.go:71] worker 12 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0504 20:40:27.098920 1 tasks_processing.go:71] worker 51 working on ingress_certificates task. I0504 20:40:27.098921 1 tasks_processing.go:71] worker 1 working on openstack_dataplanenodesets task. I0504 20:40:27.098923 1 tasks_processing.go:71] worker 34 working on nodenetworkconfigurationpolicies task. I0504 20:40:27.098928 1 tasks_processing.go:71] worker 55 working on nodes task. I0504 20:40:27.099019 1 tasks_processing.go:71] worker 52 working on support_secret task. I0504 20:40:27.098874 1 tasks_processing.go:71] worker 35 working on dvo_metrics task. I0504 20:40:27.103784 1 controller.go:129] Initializing last reported time to 0001-01-01T00:00:00Z I0504 20:40:27.103806 1 controller.go:368] The initial operator extension status is healthy I0504 20:40:27.103818 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0504 20:40:27.103824 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0504 20:40:27.103828 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0504 20:40:27.103865 1 controller.go:531] The operator is still being initialized I0504 20:40:27.103877 1 controller.go:554] The operator is healthy I0504 20:40:27.103998 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0504 20:40:27.104005 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0504 20:40:27.104119 1 operator.go:328] started I0504 20:40:27.104142 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0504 20:40:27.104750 1 tasks_processing.go:71] worker 60 working on operators task. I0504 20:40:27.104767 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 6.295886ms to process 0 records E0504 20:40:27.105058 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.166.99:9091: connect: connection refused I0504 20:40:27.105082 1 tasks_processing.go:71] worker 58 working on cost_management_metrics_configs task. E0504 20:40:27.105118 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.166.99:9091: connect: connection refused I0504 20:40:27.105136 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 5.532401ms to process 0 records E0504 20:40:27.105132 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.166.99:9091: connect: connection refused I0504 20:40:27.105153 1 tasks_processing.go:74] worker 5 stopped. I0504 20:40:27.105523 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=5779c577e5a80d2a1db752d6c204fa93e374b8052d9ca09fd5476d1046ae80b9 I0504 20:40:27.105548 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 7.165298ms to process 1 records E0504 20:40:27.105559 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.166.99:9091: connect: connection refused I0504 20:40:27.105588 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 6.928356ms to process 0 records I0504 20:40:27.105608 1 tasks_processing.go:74] worker 16 stopped. E0504 20:40:27.108718 1 gather_node_features.go:86] GatherNodeFeatures: NodeFeatures resource not found in openshift-nfd namespace (may not be installed) I0504 20:40:27.108734 1 tasks_processing.go:74] worker 37 stopped. I0504 20:40:27.108741 1 gather.go:177] gatherer "clusterconfig" function "node_features" took 9.410134ms to process 0 records I0504 20:40:27.108928 1 tasks_processing.go:74] worker 36 stopped. I0504 20:40:27.108935 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 9.466059ms to process 0 records I0504 20:40:27.109246 1 tasks_processing.go:74] worker 22 stopped. I0504 20:40:27.109262 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 11.024203ms to process 0 records I0504 20:40:27.109760 1 tasks_processing.go:74] worker 26 stopped. I0504 20:40:27.109776 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 11.204952ms to process 0 records I0504 20:40:27.110061 1 tasks_processing.go:74] worker 14 stopped. I0504 20:40:27.110253 1 recorder.go:75] Recording config/authentication with fingerprint=2695442563b9077aca0051da48a3981802b36e22039ef4b91f704c694da51075 I0504 20:40:27.110266 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 12.045502ms to process 1 records I0504 20:40:27.112926 1 tasks_processing.go:74] worker 30 stopped. I0504 20:40:27.113144 1 recorder.go:75] Recording config/apiserver with fingerprint=08aa6e35c9731ec7c8faeec027b7e1c5c388111ab1ed7b0e8677f14dc6848e89 I0504 20:40:27.113184 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 14.604377ms to process 1 records I0504 20:40:27.113200 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 14.269656ms to process 0 records I0504 20:40:27.113220 1 tasks_processing.go:74] worker 2 stopped. I0504 20:40:27.113284 1 tasks_processing.go:74] worker 56 stopped. I0504 20:40:27.113457 1 recorder.go:75] Recording config/oauth with fingerprint=e6ae0e4ad6dc22f41060aa69be275cb11d320e66b7fd0c605c4dc80680919579 I0504 20:40:27.113491 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 14.635528ms to process 1 records I0504 20:40:27.113504 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 14.409658ms to process 0 records I0504 20:40:27.113512 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 14.779513ms to process 0 records I0504 20:40:27.113520 1 tasks_processing.go:74] worker 41 stopped. I0504 20:40:27.113524 1 tasks_processing.go:74] worker 28 stopped. I0504 20:40:27.113594 1 tasks_processing.go:74] worker 7 stopped. I0504 20:40:27.113627 1 recorder.go:75] Recording config/ingress with fingerprint=ead402e5d82d4ac9a4f4ebf43abc48b1b36da2841c5bbcd7841fb7af57db3560 I0504 20:40:27.113635 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 15.046112ms to process 1 records I0504 20:40:27.113715 1 tasks_processing.go:74] worker 15 stopped. I0504 20:40:27.113729 1 recorder.go:75] Recording config/network with fingerprint=bb6d611d78e886e087f47773388f3f1bf40fefd6a337f7724626ec64f6985897 I0504 20:40:27.113735 1 gather.go:177] gatherer "clusterconfig" function "networks" took 15.397565ms to process 1 records I0504 20:40:27.114389 1 tasks_processing.go:74] worker 6 stopped. I0504 20:40:27.114402 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 16.320302ms to process 0 records I0504 20:40:27.114412 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 16.363544ms to process 0 records I0504 20:40:27.114419 1 tasks_processing.go:74] worker 8 stopped. I0504 20:40:27.114496 1 tasks_processing.go:74] worker 45 stopped. E0504 20:40:27.114514 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0504 20:40:27.114532 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 15.769879ms to process 0 records I0504 20:40:27.114581 1 tasks_processing.go:74] worker 53 stopped. I0504 20:40:27.114590 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 15.603044ms to process 0 records I0504 20:40:27.114606 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 15.547269ms to process 0 records I0504 20:40:27.114619 1 tasks_processing.go:74] worker 25 stopped. I0504 20:40:27.115139 1 tasks_processing.go:74] worker 3 stopped. I0504 20:40:27.115261 1 recorder.go:75] Recording config/featuregate with fingerprint=c8536c494edebab5549e14a22d6c98dc6a3077a030432653f7b4aee459662499 I0504 20:40:27.115275 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 17.160978ms to process 1 records I0504 20:40:27.116178 1 tasks_processing.go:74] worker 13 stopped. I0504 20:40:27.116195 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 16.955603ms to process 0 records I0504 20:40:27.118853 1 tasks_processing.go:74] worker 20 stopped. I0504 20:40:27.118874 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 20.545639ms to process 0 records I0504 20:40:27.118953 1 tasks_processing.go:74] worker 48 stopped. I0504 20:40:27.119128 1 recorder.go:75] Recording config/image with fingerprint=aadabadd7799a24bfd7e094922bff237019a57ff7fc852f4f0b1efa232090e4f I0504 20:40:27.119142 1 gather.go:177] gatherer "clusterconfig" function "image" took 20.378275ms to process 1 records I0504 20:40:27.119513 1 tasks_processing.go:74] worker 11 stopped. I0504 20:40:27.119852 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=2fbe4bf8c8a6afe294d5d1da5284f347cbb94c71ef4d68ecc50bd0eb672d62e6 I0504 20:40:27.119870 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 21.331005ms to process 1 records I0504 20:40:27.121222 1 tasks_processing.go:74] worker 33 stopped. I0504 20:40:27.121286 1 recorder.go:75] Recording config/proxy with fingerprint=8854c0802be30fbfda435e5d5b6d83c5c728d18f0dfdd515724ba14e032a635c I0504 20:40:27.121301 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 21.668588ms to process 1 records I0504 20:40:27.121758 1 gather_logs.go:145] no pods in namespace were found I0504 20:40:27.121772 1 tasks_processing.go:74] worker 17 stopped. I0504 20:40:27.121779 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 23.460148ms to process 0 records I0504 20:40:27.122687 1 tasks_processing.go:74] worker 44 stopped. I0504 20:40:27.122701 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 24.003322ms to process 0 records I0504 20:40:27.123691 1 tasks_processing.go:74] worker 21 stopped. I0504 20:40:27.123704 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 24.485811ms to process 0 records I0504 20:40:27.123715 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 24.168505ms to process 0 records I0504 20:40:27.123719 1 tasks_processing.go:74] worker 29 stopped. I0504 20:40:27.123720 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 25.132833ms to process 0 records I0504 20:40:27.123723 1 tasks_processing.go:74] worker 54 stopped. I0504 20:40:27.123769 1 tasks_processing.go:74] worker 23 stopped. I0504 20:40:27.123884 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=c597ed330a89bc1c08b5fcd4d94f5340bbee484324009cf8d40523d5d5eaed14 I0504 20:40:27.123914 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=77f9d2689baa2717a6cbf725367dbedbf13c58b1c7e09e47c2fa0b32e015d86e I0504 20:40:27.123923 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 24.756527ms to process 2 records I0504 20:40:27.123979 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=f75c4d68dc0a9390a7e4a2d49cd8dcaa1831acaebc91aa076d5894cd8f7a9846 I0504 20:40:27.123988 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 25.134356ms to process 1 records I0504 20:40:27.123993 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 24.159111ms to process 0 records I0504 20:40:27.123998 1 tasks_processing.go:74] worker 31 stopped. I0504 20:40:27.124003 1 tasks_processing.go:74] worker 9 stopped. I0504 20:40:27.126005 1 tasks_processing.go:74] worker 34 stopped. I0504 20:40:27.126018 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 25.668513ms to process 0 records I0504 20:40:27.131067 1 tasks_processing.go:74] worker 57 stopped. E0504 20:40:27.131077 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0504 20:40:27.131104 1 gather.go:177] gatherer "clusterconfig" function "machines" took 31.003598ms to process 0 records I0504 20:40:27.131173 1 tasks_processing.go:74] worker 1 stopped. I0504 20:40:27.131185 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 30.923872ms to process 0 records I0504 20:40:27.131277 1 tasks_processing.go:74] worker 58 stopped. I0504 20:40:27.131291 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 26.162326ms to process 0 records I0504 20:40:27.131462 1 tasks_processing.go:74] worker 32 stopped. I0504 20:40:27.131471 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 31.347814ms to process 0 records I0504 20:40:27.131604 1 tasks_processing.go:74] worker 18 stopped. I0504 20:40:27.131619 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 32.765276ms to process 0 records E0504 20:40:27.131632 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0504 20:40:27.131665 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0504 20:40:27.131682 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 32.656193ms to process 1 records I0504 20:40:27.131691 1 tasks_processing.go:74] worker 39 stopped. I0504 20:40:27.131733 1 tasks_processing.go:74] worker 40 stopped. I0504 20:40:27.132392 1 recorder.go:75] Recording config/infrastructure with fingerprint=a26fa61122c02346ef337b3edb82864eb7ef487fb48dc4c76ed24a1860dc5f63 I0504 20:40:27.132434 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 32.073474ms to process 1 records I0504 20:40:27.132655 1 tasks_processing.go:74] worker 49 stopped. I0504 20:40:27.132892 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=0fc561fb4abab925eaea19ee8191d28bd5053d9b7155bcb0e2039173312586ab I0504 20:40:27.132925 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=f5eaf14b5b5a5e243e2e35853cd342bfb97bdc510c0ec574902af740efb54118 I0504 20:40:27.132961 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=9cdebb49b750943df44f8e12fd22461cf5a3efd7d03f8c572fd3b4e6fd527d58 I0504 20:40:27.132970 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 32.976808ms to process 3 records I0504 20:40:27.133385 1 tasks_processing.go:74] worker 10 stopped. I0504 20:40:27.133475 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=80f1b2378548e6d0d8e65d061e50565f8f7484fe15e7d0fc6e6db45490724ab8 I0504 20:40:27.133495 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=47e727974fb7dd59a7d4e7563281db7244260d03f3ccda4879de854b4811aaaf I0504 20:40:27.133507 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=5ff6ca135289dfa570b80c0667ec194f3d3f5fca12174a251675f0f86eaa66ac I0504 20:40:27.133520 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=875e35f03e582a7fac8fb00e8dc9661ebc85f0bd8b4d27205810f03eae21b4da I0504 20:40:27.133536 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=398af974d14c9f1ae5332296eafa5714ce7fb300f99e7695e75f7534a0efc98c I0504 20:40:27.133548 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=d9608f33608e6b6545608caa57ca0a755353b821ef1b6d64eb7b47646ceae7db I0504 20:40:27.133560 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=5ef3bd42e07d2bf125c63833743ac1cd1477cf0a6119fc9b432670118342abd5 I0504 20:40:27.133574 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=380a67f2c11c5e315e8134eeddbf322b33b3770fd4c78c4fced070f80d17fb65 I0504 20:40:27.133603 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=b85dd569979ebb61eedfa920bf9787ccf94ccf4a8e30f792f945f15339aa574e I0504 20:40:27.133619 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=7175231676bc7a0b7771e3c4a3bdcfc4cb4cbca0005bc87e2b7b720ad27685d8 I0504 20:40:27.133631 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=8062b82b83574c598f91c08dba2006ce089f338291f61d624c5bbc3c0e00935c I0504 20:40:27.133642 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 35.337238ms to process 11 records I0504 20:40:27.133709 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=78c836ddcb1e7f90d17d1744670065662cc0798a287d1ddb467fe63396d10e79 I0504 20:40:27.133736 1 tasks_processing.go:74] worker 59 stopped. I0504 20:40:27.133757 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=c360cfc82516adb212c4c29d3a7d51017663d6c85c545e8b606c659a743849ec I0504 20:40:27.133832 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=e9321b606bf141f91ed06e43a6816c97db04032f938dbb80997ffaefa06ad504 I0504 20:40:27.133855 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=362d4775679491f528d2f7dda950eb171af7acdd389e3b73303182e3a8be65d8 I0504 20:40:27.133875 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=aad466e488f7f522978a1ec9b52e4807317f1e5913424ef058f7b9d3d894a4dd I0504 20:40:27.133897 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=6f0b1e6dc7675ce244af55149034e78b80d89dc11282dc4ea59fbeabf54ccfa2 I0504 20:40:27.133920 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=a6a89a1846f6266fcfbcc731a0581c132668eb8ba3706e29782b570fa32e6ef4 I0504 20:40:27.133941 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=841c941b43d22c53d9d54c93375f7361b06fd0b325a6b030d66393853b5bfb6d I0504 20:40:27.133964 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=2ee041e7c59e60520616e64cdd6ef285848df2b67bef331792267f9377df015a I0504 20:40:27.133994 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=49e89f39f86efbc9e487fc48b94753d87ce024de20022bde27b11e94cda36fb5 I0504 20:40:27.134035 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=c4c5da894c58b685014c60bc7153d073109c4518d363e4818eab06f9579fa4fe I0504 20:40:27.134063 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=fab60a49ec5a5f7791cf561ba2b171bf7ad5516eff772a10208edddc0b9db3a0 I0504 20:40:27.134111 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=88327e483a8e7871982a921e7a207da7e4a360d4a72e61c61ba842a97bd4dc45 I0504 20:40:27.134119 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 34.341192ms to process 13 records E0504 20:40:27.134130 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0504 20:40:27.134135 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 33.885863ms to process 0 records I0504 20:40:27.134141 1 tasks_processing.go:74] worker 50 stopped. I0504 20:40:27.134218 1 tasks_processing.go:74] worker 43 stopped. I0504 20:40:27.134236 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0504 20:40:27.134242 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 34.372562ms to process 1 records I0504 20:40:27.140430 1 tasks_processing.go:74] worker 63 stopped. I0504 20:40:27.140673 1 recorder.go:75] Recording config/version with fingerprint=03a76765e3a08ccfc53e3faa30d6064d6f8ee54a99928319f88a901ec3b4a0db I0504 20:40:27.140686 1 recorder.go:75] Recording config/id with fingerprint=c424e18cf507b68b9363c5286e800bf6808cc576f223b7502e93882bc7d6c488 I0504 20:40:27.140691 1 gather.go:177] gatherer "clusterconfig" function "version" took 41.316461ms to process 2 records E0504 20:40:27.140701 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0504 20:40:27.140707 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 40.075247ms to process 0 records I0504 20:40:27.140713 1 tasks_processing.go:74] worker 52 stopped. I0504 20:40:27.142025 1 tasks_processing.go:74] worker 46 stopped. I0504 20:40:27.142030 1 controller.go:254] Source scaController *sca.Controller is not ready I0504 20:40:27.142162 1 controller.go:254] Source clusterTransferController *clustertransfer.Controller is not ready I0504 20:40:27.142189 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0504 20:40:27.142212 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0504 20:40:27.142235 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0504 20:40:27.143120 1 controller.go:531] The operator is still being initialized I0504 20:40:27.143134 1 controller.go:554] The operator is healthy I0504 20:40:27.143191 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=8c6bf1f3744339492a7b73c81010480c150d345a1423f55ba76739d8c6a6c832 I0504 20:40:27.143316 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=2bbbc9f5e2a27cba5227692a9f98bbe4ad4134b58976b8d175f78a2526cafcbe I0504 20:40:27.143329 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 43.769639ms to process 2 records I0504 20:40:27.143390 1 recorder.go:75] Recording config/olm_operators with fingerprint=f479a6faac14fdbfb21b5de8f0897fd4a46ac4986b222760d38d537ef09d999b I0504 20:40:27.143403 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 43.843735ms to process 1 records I0504 20:40:27.143407 1 tasks_processing.go:74] worker 47 stopped. W0504 20:40:27.145502 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0504 20:40:27.146060 1 tasks_processing.go:74] worker 42 stopped. I0504 20:40:27.146722 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=7bea4271bbeba97292d1464dd1ad6b77ca481668c20083b2b458ccd4f8b7892d I0504 20:40:27.146968 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=a5ff4d42406ba6de79c71fe73f56239b669f60033c08a1d9c7762bdc2832e24f I0504 20:40:27.146980 1 gather.go:177] gatherer "clusterconfig" function "crds" took 46.261461ms to process 2 records I0504 20:40:27.147775 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0504 20:40:27.149562 1 tasks_processing.go:74] worker 55 stopped. I0504 20:40:27.149877 1 recorder.go:75] Recording config/node/ip-10-0-0-118.ec2.internal with fingerprint=d5b56962b594e9a1757f5a7f3e670e818dc6abf0f30f008d6b874cbb937240bc I0504 20:40:27.149946 1 recorder.go:75] Recording config/node/ip-10-0-1-197.ec2.internal with fingerprint=d203dd8254c4ad68a521b6267a3c365b7d6b2e845326f772dccd909bc09ff661 I0504 20:40:27.150000 1 recorder.go:75] Recording config/node/ip-10-0-2-227.ec2.internal with fingerprint=9e5fd7f878504e96552c26d58bb5f6886345e0be965c010f13e37fa923977093 I0504 20:40:27.150010 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 48.988021ms to process 3 records I0504 20:40:27.150403 1 tasks_processing.go:74] worker 4 stopped. I0504 20:40:27.150420 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 52.42682ms to process 0 records I0504 20:40:27.154227 1 tasks_processing.go:74] worker 38 stopped. I0504 20:40:27.154244 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 55.729459ms to process 0 records I0504 20:40:27.162461 1 tasks_processing.go:74] worker 24 stopped. I0504 20:40:27.164024 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-66dbbb6468-cd5kc with fingerprint=50786e309e2d98deb6db05524d53dac605a99515fbf8dccad0e49c15e202122a I0504 20:40:27.164167 1 recorder.go:75] Recording config/running_containers with fingerprint=5ac52ab0cbf365853c0390832336ae106fc9e61243b11970d17cfd61353d101e I0504 20:40:27.164182 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 64.226276ms to process 2 records I0504 20:40:27.167159 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0504 20:40:27.167171 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0504 20:40:27.167220 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0504 20:40:27.167592 1 tasks_processing.go:74] worker 27 stopped. E0504 20:40:27.167605 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0504 20:40:27.167611 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0504 20:40:27.167614 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0504 20:40:27.167623 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=ad286723f58bdcfc37aeba3ec5b4110c08e1af59cd34d14b4bfaab02d18e4856 I0504 20:40:27.167662 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0504 20:40:27.167685 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0504 20:40:27.167691 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=2ab5ab7b1b10d7fcf1197bb24dea7c90f400e4effc18c7356873209d54fdf84b I0504 20:40:27.167702 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0504 20:40:27.167765 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0504 20:40:27.167773 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0504 20:40:27.167778 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 69.355272ms to process 7 records I0504 20:40:27.170731 1 base_controller.go:82] Caches are synced for ConfigController I0504 20:40:27.170748 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0504 20:40:27.172536 1 tasks_processing.go:74] worker 12 stopped. I0504 20:40:27.172553 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 72.371637ms to process 0 records I0504 20:40:27.175758 1 configmapobserver.go:84] configmaps "insights-config" not found I0504 20:40:27.175777 1 tasks_processing.go:74] worker 51 stopped. E0504 20:40:27.175788 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0504 20:40:27.175794 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2q3b2nk7rpcd7ji0vj5ecnlo6j1q2dqo-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2q3b2nk7rpcd7ji0vj5ecnlo6j1q2dqo-primary-cert-bundle-secret" not found I0504 20:40:27.175854 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=f9dc162821a0cbfdd414148206a3097bc114cbd514d611b3da901ef40dd40f30 I0504 20:40:27.175867 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 75.598673ms to process 1 records I0504 20:40:27.204738 1 base_controller.go:82] Caches are synced for LoggingSyncer I0504 20:40:27.204750 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0504 20:40:27.216607 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0504 20:40:27.220617 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0504 20:40:27.245745 1 cluster_transfer.go:102] no available accepted cluster transfer I0504 20:40:27.245759 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0504 20:40:27.305290 1 sca.go:266] etc-pki-entitlement secret successfully created I0504 20:40:27.305306 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated W0504 20:40:28.144656 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0504 20:40:28.157049 1 tasks_processing.go:74] worker 61 stopped. I0504 20:40:28.157068 1 recorder.go:75] Recording config/silenced_alerts.json with fingerprint=37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570 I0504 20:40:28.157080 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 1.057680239s to process 1 records I0504 20:40:28.157234 1 tasks_processing.go:74] worker 0 stopped. I0504 20:40:28.157275 1 recorder.go:75] Recording config/alerts with fingerprint=52a19ee470212f1c133c0170df7fa572ef2151d5be37cc2c91a2ccf251f99813 I0504 20:40:28.157296 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 1.05916745s to process 1 records W0504 20:40:29.145048 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0504 20:40:29.582390 1 tasks_processing.go:74] worker 60 stopped. I0504 20:40:29.582511 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=98549e9b10324945a09428d33a7f6e7398a9f55f77403c1857c9424e52393ddc I0504 20:40:29.582559 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0504 20:40:29.582594 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=82c058742bc0d280194e0707162104ced954a533aa7fea4f1574275a3bf6bba1 I0504 20:40:29.582604 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0504 20:40:29.582624 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=1ca4080f9693d497028a1a80972479fd82e3cda723e9763d0bc2f93a07d4b0f5 I0504 20:40:29.582642 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0504 20:40:29.582661 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=ff1f79c3eeba705f73c14891cd6cb49f325e165337747ff5ae3d380f3b38a432 I0504 20:40:29.582689 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=61264b7b0f3a1c299904ac3279b8202caafac3f8cb8932c91405b530bc29da08 I0504 20:40:29.582712 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=8e053d424aa3a8f150185f93c8680c09bed6464e19b2dabb8fa64f209b2aaf8b I0504 20:40:29.582741 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=43e66f365c55ed9fbb9fe53f060667a526499ed32223b4f8d28d7c7e931d10bd I0504 20:40:29.583012 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=3a4577cccd8ab4b84205276cba053f55dd3c3c4ba6b29efe661b0bec8ae22acc I0504 20:40:29.583034 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=e3070de4245ae26fd34a8939d7b53a569eed33a43158fadcdbfdd05362a868ec I0504 20:40:29.583069 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0504 20:40:29.583104 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=09980ba0442a1c21681e9596c7624de97acfab823cf51991266d3d6330f45a06 I0504 20:40:29.583113 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0504 20:40:29.583130 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=aae6af1e46fbe103040e40f409e37604b6ea35ee04693cd3b1e4cd4bc2a6a2e1 I0504 20:40:29.583140 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0504 20:40:29.583159 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=f47c5bfaff698273fa8efdbd1562a7c4f6231c5710a4392e138529d3ebc25ce7 I0504 20:40:29.583168 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0504 20:40:29.583188 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=67021e5de4a64d3a7a5d06f08f08c605e99909a7ca9aa550bd96fa5dccab399c I0504 20:40:29.583292 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=d2edb92a84742a26507e4867a215731f66a5f0d60907843c9e543a5ea2703e58 I0504 20:40:29.583302 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0504 20:40:29.583316 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0504 20:40:29.583338 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0504 20:40:29.583361 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=a537db1ffca1c75161cd5949763d7422f00a0e63ae790048d504df0022a390b8 I0504 20:40:29.583381 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=b62971679fe46701f81d51d946e7c211ccd1d860fdeb233a323acc1e8d703da9 I0504 20:40:29.583391 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0504 20:40:29.583406 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=e1f91a7545dfe73d6c36421e4246849b0d15a6e9b396834d3f9b1f2f51fa2427 I0504 20:40:29.583416 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0504 20:40:29.583433 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=5c30ca0f14d7cd3a00b1f4ecacdfbb6705104c763108d2066b3d1d8eca8d659d I0504 20:40:29.583443 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0504 20:40:29.583456 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=b5b1f9c92434daaa43b2340a8c39b53e7569bf72725c5b3010b9081e6575e1b7 I0504 20:40:29.583470 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=41e791290ee585c0f82118dcb29037d4db3124aa064b02c75feb5557584c8fc7 I0504 20:40:29.583485 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=fcc7e47cc5d39db86d6a00176e29900e27ff098d87e8b1cb2c6c1826cb2df16f I0504 20:40:29.583505 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=092ddce85ea4724a8caa404d880904387dc6c18e837f7a60b34fc75c4dc1742f I0504 20:40:29.583514 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0504 20:40:29.583543 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=ceba79e0b682af6e020cd8fc36bed6f1b49e07bdadde6a13d3c8c3940f7dea54 I0504 20:40:29.583561 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0504 20:40:29.583567 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0504 20:40:29.583574 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.477617171s to process 39 records W0504 20:40:30.144983 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0504 20:40:31.144633 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0504 20:40:32.142562 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0504 20:40:32.142587 1 tasks_processing.go:74] worker 35 stopped. E0504 20:40:32.142597 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0504 20:40:32.142607 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0504 20:40:32.142616 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.041993685s to process 1 records I0504 20:40:39.931326 1 tasks_processing.go:74] worker 19 stopped. I0504 20:40:39.931372 1 recorder.go:75] Recording config/installplans with fingerprint=95dfd5f33a9a46199239158a0ec0183b629818eca24ae41f4d706d7a3d604aec I0504 20:40:39.931385 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.833257806s to process 1 records I0504 20:40:40.706497 1 tasks_processing.go:74] worker 62 stopped. I0504 20:40:40.706770 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=796bd9f20e825fd32af435d21e60c10c3ec4910fa1211c5a6338f5b81c3f8071 I0504 20:40:40.706788 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.608054432s to process 1 records E0504 20:40:40.706850 1 periodic.go:254] "Unhandled Error" err="clusterconfig failed after 13.608s with: function \"tsdb_status\" failed with an error, function \"metrics\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"machines\" failed with an error, function \"machine_configs\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"support_secret\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0504 20:40:40.707961 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "tsdb_status" failed with an error, function "metrics" failed with an error, function "machine_healthchecks" failed with an error, function "machines" failed with an error, function "machine_configs" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "support_secret" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0504 20:40:40.707976 1 periodic.go:216] Running workloads gatherer I0504 20:40:40.707994 1 tasks_processing.go:45] number of workers: 2 I0504 20:40:40.708006 1 tasks_processing.go:69] worker 1 listening for tasks. I0504 20:40:40.708010 1 tasks_processing.go:71] worker 1 working on workload_info task. I0504 20:40:40.708020 1 tasks_processing.go:69] worker 0 listening for tasks. I0504 20:40:40.708045 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0504 20:40:40.715451 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-118.ec2.internal... I0504 20:40:40.715464 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-227.ec2.internal... I0504 20:40:40.715456 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-197.ec2.internal... I0504 20:40:40.739972 1 tasks_processing.go:74] worker 0 stopped. I0504 20:40:40.739989 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 31.906869ms to process 0 records I0504 20:40:45.264633 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 5s I0504 20:40:45.288298 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0504 20:40:45.297615 1 gather_workloads_info.go:387] No image sha256:730d1b6988025bef0daa3a9a5d8467ec4a26b0382cc52f91c3375b4590d3518a (11ms) I0504 20:40:45.306796 1 gather_workloads_info.go:387] No image sha256:a043239802b3eb8b323d285193d2527fad0ecec98ca91d188a3472a2fac8ae04 (9ms) I0504 20:40:45.317365 1 gather_workloads_info.go:387] No image sha256:6814f5fced0ef219d06374011c68a11a3da788a764a00a69ade435466d9ee240 (11ms) I0504 20:40:45.327302 1 gather_workloads_info.go:387] No image sha256:15677f0b70e6aa2dfaf088e45fc1a425c22bf6fda326b8116f87e88b6694dfab (10ms) I0504 20:40:45.337204 1 gather_workloads_info.go:387] No image sha256:d6fbe0075cbb12bfd287c973704fadf97154c7f73e370733d976a40835e9155a (10ms) I0504 20:40:45.346883 1 gather_workloads_info.go:387] No image sha256:53ff8efdd8f563421ebc1859ab021a3bbc1306f82d4ff5d9c21bc60ec5f57898 (10ms) I0504 20:40:45.356015 1 gather_workloads_info.go:387] No image sha256:1eb6a6ee3c66ac15be6bc21bb6015c595c11c15b9dfd21bd9f5ce7f7b990ebaa (9ms) I0504 20:40:45.365773 1 gather_workloads_info.go:387] No image sha256:89277d8d4560d71db88c2dcc67c992a24544ca21810920b609c1d49d53b4a287 (10ms) I0504 20:40:45.377067 1 gather_workloads_info.go:387] No image sha256:5ff204630794311b0b37fa7b197b933ad85d76a481bf7cdb3bcbada08f0cdcbf (11ms) I0504 20:40:45.386795 1 gather_workloads_info.go:387] No image sha256:572ebf7c7fab1a518582f3ca46c866730423cce6e8a19f614dc15fd61bf0c65a (10ms) I0504 20:40:45.397568 1 gather_workloads_info.go:387] No image sha256:5aaea0419169e55832cc27acfe0fe3b9513a343d6bdf71d3da1575ed322245d0 (11ms) I0504 20:40:45.498022 1 gather_workloads_info.go:387] No image sha256:0acfe677ed58031f8381cb8a4f2d8917ec4cfa95485e2a46529dffae54c75ace (100ms) I0504 20:40:45.607318 1 gather_workloads_info.go:387] No image sha256:5c6d21c3f97366bc7ab57031cc027b67405a684bd804ce364ed5998b0685eaca (109ms) I0504 20:40:45.698644 1 gather_workloads_info.go:387] No image sha256:52db05084f2a2e7d7bf4040e4865df88ec4074efbbe59a78d861be2c5ab2aa40 (91ms) I0504 20:40:45.797741 1 gather_workloads_info.go:387] No image sha256:000105ef5150e7079b90a613fb9e6193e2a6ef9b1908d2dce44f2395d4fd070f (99ms) I0504 20:40:45.898366 1 gather_workloads_info.go:387] No image sha256:8f881f6f0498eaba860d711dc8e9dc066db8728f22abedb82e1dc9a8e1c30046 (101ms) I0504 20:40:45.997609 1 gather_workloads_info.go:387] No image sha256:934d8e8c50f3c609b8eea80d1051111fe3d066fe8c65c79572072ae55fcb0a86 (99ms) I0504 20:40:46.097951 1 gather_workloads_info.go:387] No image sha256:25c148fd380b1a9db3f6039d2e0eabc489a954921452391390ba9192b2325678 (100ms) I0504 20:40:46.196951 1 gather_workloads_info.go:387] No image sha256:0b2e6536a0ca3afc5e64a218087a6287c74b77a359075a385f1949ca5bd51b22 (99ms) I0504 20:40:46.297206 1 gather_workloads_info.go:387] No image sha256:15e1b5f763553e86d0230a751b0783e3b990438fd07cf96eed164bccb72ed564 (100ms) I0504 20:40:46.397794 1 gather_workloads_info.go:387] No image sha256:55b1db6038c5beaed54c626e3343b7a8589cc0be8dc41d1a66b4deab766ff520 (101ms) I0504 20:40:46.498391 1 gather_workloads_info.go:387] No image sha256:ab60623bb32f7e75fca71ef65137731cae347a21c7a4091dfd583fa00732721c (101ms) I0504 20:40:46.597975 1 gather_workloads_info.go:387] No image sha256:9d6d63dce784df5863e6a0ba4c11317a26c9d756d4e62d47759c97190a71cce5 (100ms) I0504 20:40:46.697512 1 gather_workloads_info.go:387] No image sha256:5d6a2b8d2508c7da71192661e7c0dcf1335f48b6336a20189abffbe79eeff962 (100ms) I0504 20:40:46.815649 1 gather_workloads_info.go:387] No image sha256:637d41f067a5239096fc22b135181cda5113da833f1370e7a73965e83792e93a (118ms) I0504 20:40:46.904271 1 gather_workloads_info.go:387] No image sha256:3bba1358d4a0ae878ff491c0c2cbfffe60649e110b40342b878fe8fa332f8858 (89ms) I0504 20:40:47.003982 1 gather_workloads_info.go:387] No image sha256:3380fecebaeee139c1607fbcd9346cebc92806679568e2244a8905ef0f52aad2 (100ms) I0504 20:40:47.108006 1 gather_workloads_info.go:398] Found image sha256:23204877f166455ce8f9de9b91a467eeda576f172d5f96124a68aa986c4775d6 (104ms) I0504 20:40:47.198474 1 gather_workloads_info.go:387] No image sha256:084aa9b0f8a6d478549dc384d4e66da13ee9b25cc98531da861cc19dee2a9e8f (90ms) I0504 20:40:47.298719 1 gather_workloads_info.go:387] No image sha256:17aefdc2f4752c23dc8f0947bb01fc34635822eeb18f2d8a5eddf8211b84d450 (100ms) I0504 20:40:47.398430 1 gather_workloads_info.go:387] No image sha256:875c77e5d144f03fb91d8cee0259f6966683ca88d1bf818dbf4652c16b70312c (100ms) I0504 20:40:47.498116 1 gather_workloads_info.go:387] No image sha256:9ef6dc311b880a6442b7a3a19006ddd955c7da36850bf000f5c9b68fd0862dd4 (100ms) I0504 20:40:47.598015 1 gather_workloads_info.go:387] No image sha256:d64bea34bf3e1bb0b3a701c3ff14e66665afc1b050f28124ad7e6888eaec3a81 (100ms) I0504 20:40:47.697630 1 gather_workloads_info.go:387] No image sha256:48e883a3932aea9457f55cd4628d21397d429d4828f42f95d1c903d9d5395bde (100ms) I0504 20:40:47.802499 1 gather_workloads_info.go:387] No image sha256:5b6212b8f539f08e78417d8a4b7485ca0b4e7927cacd7b752742a28841bc8ccd (105ms) I0504 20:40:47.898218 1 gather_workloads_info.go:387] No image sha256:30e597ec5d6bb96ff70a4f8688c748b659cd4fd5d73d222e8701821d236795c5 (96ms) I0504 20:40:47.998032 1 gather_workloads_info.go:387] No image sha256:bdeb1c455cffccdb6c90db238e9d371de06340b1d6105d47f66d87293f878ac1 (100ms) I0504 20:40:48.098798 1 gather_workloads_info.go:387] No image sha256:48f5559439bea030a6423652da448c25fff0c62ba293eb521843c1a8eb89ddb0 (101ms) I0504 20:40:48.198206 1 gather_workloads_info.go:387] No image sha256:5ac9c549d65fc1d8bc900773bebee43e9192bcec1bb5fa46afb4597230c16ac7 (99ms) I0504 20:40:48.298194 1 gather_workloads_info.go:387] No image sha256:6c6d412ccf0c3d6a14645a79df6c4658c0f8725d7954cb83c3d2f0116499345a (100ms) I0504 20:40:48.397350 1 gather_workloads_info.go:387] No image sha256:ae7d3453fd734ecc865e5f9bb16f29244ebffe6291b77e1d4e496f71eb053174 (99ms) I0504 20:40:48.498838 1 gather_workloads_info.go:387] No image sha256:0a4dfb8d4c1b3849319d45b4c54dff26a7238a2c08fcaa121f93073e95ab12e8 (101ms) I0504 20:40:48.598651 1 gather_workloads_info.go:387] No image sha256:08c5a78c8a5af04c549e2273aaf4bb452a75bf038d68aa9d01bb2aff66c30e90 (100ms) I0504 20:40:48.698581 1 gather_workloads_info.go:387] No image sha256:e1ba458cf6f0b3606c90880da72db8ab99cd11040bae84baebe3ff2e0d1ea075 (100ms) I0504 20:40:48.799395 1 gather_workloads_info.go:398] Found image sha256:673ebc8cc22c56c8f410e011b2fa950d28cf7b6420e17fdb6580d6cb10523384 (101ms) I0504 20:40:48.899439 1 gather_workloads_info.go:387] No image sha256:15928a5b803fa00781b408dbc4c3bfa7c5287b18584ba31f817def8a4430c725 (100ms) I0504 20:40:48.998066 1 gather_workloads_info.go:387] No image sha256:5808401268394502d335281ea1a294b07210461b986b58f91d7d1f29c0029c6d (99ms) I0504 20:40:48.998121 1 tasks_processing.go:74] worker 1 stopped. I0504 20:40:48.998418 1 recorder.go:75] Recording config/workload_info with fingerprint=b3dc2c0dd2d78a429ef51b26362775d1651906dba9ba49956a40f11afda48057 I0504 20:40:48.998437 1 gather.go:177] gatherer "workloads" function "workload_info" took 8.290102604s to process 1 records I0504 20:40:48.998452 1 periodic.go:263] Periodic gather workloads completed in 8.29s I0504 20:40:48.998460 1 controllerstatus.go:80] name=periodic-workloads healthy=true reason= message= I0504 20:40:48.998464 1 periodic.go:216] Running conditional gatherer I0504 20:40:49.005235 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.21.9/gathering_rules I0504 20:40:49.161529 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer I0504 20:40:50.250800 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0504 20:40:50.259794 1 conditional_gatherer.go:392] cluster version is '4.21.9' I0504 20:40:50.259822 1 tasks_processing.go:45] number of workers: 3 I0504 20:40:50.259839 1 tasks_processing.go:69] worker 2 listening for tasks. I0504 20:40:50.259844 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0504 20:40:50.259859 1 tasks_processing.go:69] worker 0 listening for tasks. I0504 20:40:50.259876 1 tasks_processing.go:69] worker 1 listening for tasks. I0504 20:40:50.259888 1 tasks_processing.go:71] worker 1 working on conditional_gatherer_rules task. I0504 20:40:50.259898 1 tasks_processing.go:74] worker 1 stopped. I0504 20:40:50.259902 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0504 20:40:50.260000 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=c8d4892cd4f2eea27d57353c6055eeb530bc2d811bfe041f6984d90395255a8c I0504 20:40:50.260021 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.948µs to process 1 records I0504 20:40:50.260067 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0504 20:40:50.260079 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.41µs to process 1 records I0504 20:40:50.260104 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 227.65µs to process 0 records I0504 20:40:50.260106 1 tasks_processing.go:74] worker 0 stopped. I0504 20:40:50.260114 1 tasks_processing.go:74] worker 2 stopped. I0504 20:40:50.260145 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0504 20:40:50.260170 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0504 20:40:50.289413 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=26c1d2451f2d97487ea9d58c0c143cc83047b9ebb0e3ff78ea02f26c1aa0e48d I0504 20:40:50.289537 1 diskrecorder.go:70] Writing 111 records to /var/lib/insights-operator/insights-2026-05-04-204050.tar.gz I0504 20:40:50.295967 1 diskrecorder.go:51] Wrote 111 records to disk in 6ms I0504 20:40:50.295998 1 periodic.go:285] Gathering cluster info every 2h0m0s I0504 20:40:50.296013 1 periodic.go:286] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0504 20:41:57.098550 1 diskrecorder.go:223] Found files to send: insights-2026-05-04-204050.tar.gz I0504 20:41:57.098581 1 insightsuploader.go:150] Checking archives to upload periodically every 15m44.073212081s I0504 20:41:57.098591 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0504 20:41:57.109430 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0504 20:41:57.276171 1 requests.go:87] Successfully reported id=2026-05-04T20:41:57Z x-rh-insights-request-id=280f7eb939124e20bf4b177f0450ad8c, wrote=48769 I0504 20:41:57.276292 1 insightsuploader.go:187] Uploaded report successfully in 177.693263ms I0504 20:41:57.276317 1 controller.go:129] Initializing last reported time to 2026-05-04T20:41:57Z I0504 20:41:57.276387 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0504 20:41:57.276396 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0504 20:41:57.276403 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0504 20:41:57.284284 1 controller.go:554] The operator is healthy I0504 20:42:27.113991 1 controller.go:554] The operator is healthy I0504 20:43:00.536195 1 insightsreport.go:137] Pulling report from smart-proxy I0504 20:43:00.536227 1 insightsreport.go:149] Retrieving report I0504 20:43:00.543487 1 requests.go:110] Retrieving report for cluster: 7c25c79e-f360-464a-94b3-323253d84dad I0504 20:43:00.543499 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/7c25c79e-f360-464a-94b3-323253d84dad/reports I0504 20:43:00.546544 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/7c25c79e-f360-464a-94b3-323253d84dad/reports I0504 20:43:00.745882 1 insightsreport.go:184] Report retrieved I0504 20:43:00.772599 1 insightsreport.go:239] Report retrieved correctly I0504 20:44:27.114632 1 controller.go:554] The operator is healthy I0504 20:45:27.091185 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0504 20:45:27.100493 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 20:45:27.100512 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0504 20:45:27.105487 1 secretconfigobserver.go:119] support secret does not exist I0504 20:46:27.112026 1 controller.go:554] The operator is healthy I0504 20:48:27.114416 1 controller.go:554] The operator is healthy I0504 20:50:27.089332 1 configmapobserver.go:84] configmaps "insights-config" not found I0504 20:50:27.106558 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0504 20:50:27.110598 1 controller.go:554] The operator is healthy I0504 20:50:27.111747 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 20:50:27.111759 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0504 20:50:27.116069 1 secretconfigobserver.go:119] support secret does not exist I0504 20:52:27.111008 1 controller.go:554] The operator is healthy I0504 20:54:27.114298 1 controller.go:554] The operator is healthy I0504 20:55:27.116128 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0504 20:55:27.125403 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 20:55:27.125422 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0504 20:55:27.129732 1 secretconfigobserver.go:119] support secret does not exist I0504 20:56:27.114385 1 controller.go:554] The operator is healthy I0504 20:57:46.351265 1 insightsuploader.go:146] Nothing to report since 2026-05-04T20:41:57Z I0504 20:58:27.114615 1 controller.go:554] The operator is healthy I0504 21:00:27.091801 1 configmapobserver.go:84] configmaps "insights-config" not found I0504 21:00:27.110382 1 controller.go:554] The operator is healthy I0504 21:00:27.130184 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0504 21:00:27.134806 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 21:00:27.134829 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0504 21:00:27.139198 1 secretconfigobserver.go:119] support secret does not exist I0504 21:02:27.111024 1 controller.go:554] The operator is healthy I0504 21:04:27.110604 1 controller.go:554] The operator is healthy I0504 21:05:27.139925 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0504 21:05:27.145815 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 21:05:27.145836 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0504 21:05:27.150625 1 secretconfigobserver.go:119] support secret does not exist I0504 21:06:27.110713 1 controller.go:554] The operator is healthy I0504 21:08:27.110782 1 controller.go:554] The operator is healthy I0504 21:10:27.089387 1 configmapobserver.go:84] configmaps "insights-config" not found I0504 21:10:27.093877 1 configmapobserver.go:84] configmaps "insights-config" not found I0504 21:10:27.110616 1 controller.go:554] The operator is healthy I0504 21:10:27.151574 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0504 21:10:27.156186 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 21:10:27.156202 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0504 21:10:27.160299 1 secretconfigobserver.go:119] support secret does not exist I0504 21:12:27.111874 1 controller.go:554] The operator is healthy