I0420 07:43:25.838405 1 cmd.go:253] Using service-serving-cert provided certificates I0420 07:43:25.838839 1 observer_polling.go:159] Starting file observer I0420 07:43:26.273095 1 operator.go:76] Starting insights-operator v0.0.0-master+$Format:%H$ I0420 07:43:26.273294 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0420 07:43:26.273845 1 secure_serving.go:57] Forcing use of http/1.1 only W0420 07:43:26.273865 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0420 07:43:26.273870 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0420 07:43:26.273874 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0420 07:43:26.273877 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0420 07:43:26.273880 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0420 07:43:26.273885 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0420 07:43:26.273949 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0420 07:43:26.278158 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"9f3affe7-8891-46aa-8817-a7df00d72437", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "HyperShiftOnlyDynamicResourceAllocation", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryption", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLM", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "OpenShiftPodSecurityAdmission", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}} I0420 07:43:26.278177 1 operator.go:141] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNSInstall GatewayAPI GatewayAPIController HighlyAvailableArbiter HyperShiftOnlyDynamicResourceAllocation ImageStreamImportMode ImageVolume KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS ManagedBootImagesAzure ManagedBootImagesvSphere MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages PreconfiguredUDNAddresses ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SigstoreImageVerification SigstoreImageVerificationPKI StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSDualStackInstall AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureDualStackInstall AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement CBORServingAndStorage CRDCompatibilityRequirementOperator ClientsAllowCBOR ClientsPreferCBOR ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterAPIMachineManagement ClusterAPIMachineManagementVSphere ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud EtcdBackendQuota EventTTL EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall GCPDualStackInstall ImageModeStatusReporting IngressControllerDynamicConfigurationManager InsightsConfig InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryption KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesCPMS MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutableCSINodeAllocatableCount MutatingAdmissionPolicy NewOLM NewOLMBoxCutterRuntime NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterInstall NutanixMultiSubnets OSStreams OVNObservability OnPremDNSRecords OpenShiftPodSecurityAdmission ProvisioningRequestAvailable SELinuxMount ShortCertRotation SignatureStores TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeGroupSnapshot] I0420 07:43:26.279781 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0420 07:43:26.279803 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0420 07:43:26.279804 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0420 07:43:26.279808 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0420 07:43:26.279818 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0420 07:43:26.279823 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0420 07:43:26.280050 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0420 07:43:26.280267 1 secure_serving.go:213] Serving securely on [::]:8443 I0420 07:43:26.280293 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0420 07:43:26.291807 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0420 07:43:26.291830 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0420 07:43:26.291948 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0420 07:43:26.297730 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 07:43:26.297747 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0420 07:43:26.303321 1 secretconfigobserver.go:119] support secret does not exist I0420 07:43:26.309011 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 07:43:26.314621 1 secretconfigobserver.go:119] support secret does not exist I0420 07:43:26.317466 1 recorder.go:176] Pruning old reports every 5h33m3s, max age is 288h0m0s I0420 07:43:26.323357 1 periodic.go:216] Running clusterconfig gatherer I0420 07:43:26.323361 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0420 07:43:26.323376 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0420 07:43:26.323398 1 tasks_processing.go:45] number of workers: 64 I0420 07:43:26.323405 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0420 07:43:26.323409 1 insightsreport.go:296] Starting report retriever I0420 07:43:26.323413 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0420 07:43:26.323413 1 tasks_processing.go:69] worker 0 listening for tasks. I0420 07:43:26.323422 1 tasks_processing.go:69] worker 9 listening for tasks. I0420 07:43:26.323424 1 tasks_processing.go:69] worker 4 listening for tasks. I0420 07:43:26.323428 1 tasks_processing.go:69] worker 5 listening for tasks. I0420 07:43:26.323430 1 tasks_processing.go:69] worker 1 listening for tasks. I0420 07:43:26.323434 1 tasks_processing.go:69] worker 6 listening for tasks. I0420 07:43:26.323435 1 tasks_processing.go:69] worker 2 listening for tasks. I0420 07:43:26.323438 1 tasks_processing.go:71] worker 6 working on node_features task. I0420 07:43:26.323437 1 tasks_processing.go:71] worker 1 working on image_pruners task. I0420 07:43:26.323443 1 tasks_processing.go:69] worker 10 listening for tasks. I0420 07:43:26.323449 1 tasks_processing.go:69] worker 11 listening for tasks. I0420 07:43:26.323454 1 tasks_processing.go:69] worker 12 listening for tasks. I0420 07:43:26.323459 1 tasks_processing.go:69] worker 13 listening for tasks. I0420 07:43:26.323464 1 tasks_processing.go:69] worker 14 listening for tasks. I0420 07:43:26.323470 1 tasks_processing.go:69] worker 42 listening for tasks. I0420 07:43:26.323476 1 tasks_processing.go:69] worker 43 listening for tasks. I0420 07:43:26.323481 1 tasks_processing.go:69] worker 44 listening for tasks. I0420 07:43:26.323479 1 tasks_processing.go:69] worker 41 listening for tasks. I0420 07:43:26.323488 1 tasks_processing.go:69] worker 45 listening for tasks. I0420 07:43:26.323491 1 tasks_processing.go:69] worker 28 listening for tasks. I0420 07:43:26.323495 1 tasks_processing.go:69] worker 27 listening for tasks. I0420 07:43:26.323493 1 tasks_processing.go:69] worker 46 listening for tasks. I0420 07:43:26.323497 1 tasks_processing.go:69] worker 47 listening for tasks. I0420 07:43:26.323502 1 tasks_processing.go:69] worker 29 listening for tasks. I0420 07:43:26.323502 1 tasks_processing.go:69] worker 48 listening for tasks. I0420 07:43:26.323507 1 tasks_processing.go:69] worker 59 listening for tasks. I0420 07:43:26.323506 1 tasks_processing.go:69] worker 30 listening for tasks. I0420 07:43:26.323511 1 tasks_processing.go:69] worker 60 listening for tasks. I0420 07:43:26.323513 1 tasks_processing.go:69] worker 31 listening for tasks. I0420 07:43:26.323516 1 tasks_processing.go:69] worker 61 listening for tasks. I0420 07:43:26.323517 1 tasks_processing.go:69] worker 58 listening for tasks. I0420 07:43:26.323521 1 tasks_processing.go:69] worker 62 listening for tasks. I0420 07:43:26.323523 1 tasks_processing.go:69] worker 32 listening for tasks. I0420 07:43:26.323528 1 tasks_processing.go:71] worker 4 working on cost_management_metrics_configs task. I0420 07:43:26.323530 1 tasks_processing.go:69] worker 33 listening for tasks. I0420 07:43:26.323526 1 tasks_processing.go:69] worker 49 listening for tasks. I0420 07:43:26.323534 1 tasks_processing.go:69] worker 50 listening for tasks. I0420 07:43:26.323535 1 tasks_processing.go:69] worker 15 listening for tasks. I0420 07:43:26.323540 1 tasks_processing.go:69] worker 16 listening for tasks. I0420 07:43:26.323541 1 tasks_processing.go:69] worker 51 listening for tasks. I0420 07:43:26.323548 1 tasks_processing.go:69] worker 52 listening for tasks. I0420 07:43:26.323540 1 tasks_processing.go:71] worker 9 working on sap_config task. I0420 07:43:26.323550 1 tasks_processing.go:71] worker 5 working on machine_configs task. I0420 07:43:26.323559 1 tasks_processing.go:69] worker 53 listening for tasks. I0420 07:43:26.323559 1 tasks_processing.go:69] worker 35 listening for tasks. I0420 07:43:26.323564 1 tasks_processing.go:69] worker 25 listening for tasks. I0420 07:43:26.323560 1 tasks_processing.go:69] worker 24 listening for tasks. I0420 07:43:26.323568 1 tasks_processing.go:69] worker 54 listening for tasks. I0420 07:43:26.323552 1 tasks_processing.go:69] worker 34 listening for tasks. I0420 07:43:26.323574 1 tasks_processing.go:69] worker 36 listening for tasks. I0420 07:43:26.323575 1 tasks_processing.go:69] worker 26 listening for tasks. I0420 07:43:26.323553 1 tasks_processing.go:69] worker 23 listening for tasks. I0420 07:43:26.323585 1 tasks_processing.go:69] worker 37 listening for tasks. I0420 07:43:26.323587 1 tasks_processing.go:69] worker 18 listening for tasks. I0420 07:43:26.323591 1 tasks_processing.go:69] worker 56 listening for tasks. I0420 07:43:26.323593 1 tasks_processing.go:69] worker 3 listening for tasks. I0420 07:43:26.323585 1 tasks_processing.go:69] worker 55 listening for tasks. I0420 07:43:26.323601 1 tasks_processing.go:69] worker 19 listening for tasks. I0420 07:43:26.323570 1 tasks_processing.go:69] worker 22 listening for tasks. I0420 07:43:26.323594 1 tasks_processing.go:69] worker 38 listening for tasks. I0420 07:43:26.323602 1 tasks_processing.go:69] worker 39 listening for tasks. I0420 07:43:26.323611 1 tasks_processing.go:69] worker 40 listening for tasks. I0420 07:43:26.323608 1 tasks_processing.go:69] worker 7 listening for tasks. I0420 07:43:26.323548 1 tasks_processing.go:69] worker 17 listening for tasks. I0420 07:43:26.323602 1 tasks_processing.go:69] worker 57 listening for tasks. I0420 07:43:26.323610 1 tasks_processing.go:69] worker 20 listening for tasks. I0420 07:43:26.323625 1 tasks_processing.go:71] worker 2 working on nodenetworkconfigurationpolicies task. I0420 07:43:26.323632 1 tasks_processing.go:71] worker 28 working on olm_operators task. I0420 07:43:26.323613 1 tasks_processing.go:69] worker 8 listening for tasks. I0420 07:43:26.323633 1 tasks_processing.go:71] worker 10 working on sap_datahubs task. I0420 07:43:26.323628 1 tasks_processing.go:71] worker 11 working on openstack_dataplanenodesets task. I0420 07:43:26.323525 1 tasks_processing.go:69] worker 63 listening for tasks. I0420 07:43:26.323864 1 tasks_processing.go:71] worker 0 working on aggregated_monitoring_cr_names task. I0420 07:43:26.323901 1 tasks_processing.go:71] worker 60 working on openstack_dataplanedeployments task. I0420 07:43:26.323972 1 tasks_processing.go:71] worker 63 working on ingress_certificates task. I0420 07:43:26.324017 1 tasks_processing.go:71] worker 29 working on authentication task. I0420 07:43:26.324098 1 tasks_processing.go:71] worker 44 working on operators_pods_and_events task. I0420 07:43:26.324172 1 tasks_processing.go:71] worker 43 working on openshift_machine_api_events task. I0420 07:43:26.323643 1 tasks_processing.go:71] worker 34 working on container_runtime_configs task. I0420 07:43:26.324065 1 tasks_processing.go:71] worker 46 working on proxies task. I0420 07:43:26.324398 1 tasks_processing.go:71] worker 42 working on pod_network_connectivity_checks task. I0420 07:43:26.324403 1 tasks_processing.go:71] worker 35 working on lokistack task. I0420 07:43:26.324062 1 tasks_processing.go:71] worker 47 working on validating_webhook_configurations task. I0420 07:43:26.324806 1 tasks_processing.go:71] worker 52 working on jaegers task. I0420 07:43:26.324862 1 tasks_processing.go:71] worker 53 working on active_alerts task. I0420 07:43:26.324075 1 tasks_processing.go:71] worker 13 working on networks task. I0420 07:43:26.324941 1 tasks_processing.go:71] worker 41 working on metrics task. I0420 07:43:26.325004 1 tasks_processing.go:71] worker 58 working on support_secret task. I0420 07:43:26.325111 1 tasks_processing.go:71] worker 26 working on oauths task. I0420 07:43:26.325156 1 tasks_processing.go:71] worker 40 working on feature_gates task. I0420 07:43:26.325169 1 tasks_processing.go:71] worker 57 working on silenced_alerts task. I0420 07:43:26.325176 1 tasks_processing.go:71] worker 18 working on machine_autoscalers task. I0420 07:43:26.325202 1 tasks_processing.go:71] worker 37 working on machine_healthchecks task. I0420 07:43:26.325397 1 tasks_processing.go:71] worker 49 working on dvo_metrics task. I0420 07:43:26.325438 1 tasks_processing.go:71] worker 62 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0420 07:43:26.325491 1 tasks_processing.go:71] worker 33 working on pdbs task. I0420 07:43:26.325580 1 tasks_processing.go:71] worker 55 working on openshift_logging task. I0420 07:43:26.325645 1 tasks_processing.go:71] worker 16 working on machine_sets task. I0420 07:43:26.325737 1 tasks_processing.go:71] worker 8 working on tsdb_status task. I0420 07:43:26.325399 1 tasks_processing.go:71] worker 22 working on openstack_version task. I0420 07:43:26.325634 1 tasks_processing.go:71] worker 15 working on image_registries task. I0420 07:43:26.325457 1 tasks_processing.go:71] worker 20 working on storage_classes task. I0420 07:43:26.324950 1 tasks_processing.go:71] worker 38 working on monitoring_persistent_volumes task. I0420 07:43:26.325167 1 tasks_processing.go:71] worker 23 working on container_images task. I0420 07:43:26.324901 1 tasks_processing.go:71] worker 51 working on image task. I0420 07:43:26.324091 1 tasks_processing.go:71] worker 14 working on infrastructures task. I0420 07:43:26.323630 1 tasks_processing.go:69] worker 21 listening for tasks. I0420 07:43:26.326364 1 tasks_processing.go:71] worker 21 working on machines task. I0420 07:43:26.325079 1 tasks_processing.go:71] worker 61 working on storage_cluster task. I0420 07:43:26.325092 1 tasks_processing.go:71] worker 56 working on config_maps task. I0420 07:43:26.325223 1 tasks_processing.go:71] worker 45 working on nodes task. I0420 07:43:26.325231 1 tasks_processing.go:71] worker 39 working on machine_config_pools task. I0420 07:43:26.325247 1 tasks_processing.go:71] worker 7 working on mutating_webhook_configurations task. I0420 07:43:26.325257 1 tasks_processing.go:71] worker 48 working on sap_pods task. I0420 07:43:26.325303 1 tasks_processing.go:71] worker 54 working on node_logs task. I0420 07:43:26.325312 1 tasks_processing.go:71] worker 3 working on openstack_controlplanes task. I0420 07:43:26.325312 1 tasks_processing.go:71] worker 24 working on ingress task. I0420 07:43:26.325326 1 tasks_processing.go:71] worker 31 working on crds task. I0420 07:43:26.325320 1 tasks_processing.go:71] worker 25 working on qemu_kubevirt_launcher_logs task. I0420 07:43:26.325344 1 tasks_processing.go:71] worker 19 working on certificate_signing_requests task. I0420 07:43:26.325349 1 tasks_processing.go:71] worker 59 working on overlapping_namespace_uids task. I0420 07:43:26.325385 1 tasks_processing.go:71] worker 17 working on operators task. I0420 07:43:26.325412 1 tasks_processing.go:71] worker 32 working on install_plans task. I0420 07:43:26.325467 1 tasks_processing.go:71] worker 50 working on version task. I0420 07:43:26.325925 1 tasks_processing.go:71] worker 30 working on schedulers task. I0420 07:43:26.323641 1 tasks_processing.go:71] worker 27 working on nodenetworkstates task. I0420 07:43:26.323646 1 tasks_processing.go:71] worker 36 working on clusterroles task. E0420 07:43:26.327799 1 gather_node_features.go:86] GatherNodeFeatures: NodeFeatures resource not found in openshift-nfd namespace (may not be installed) I0420 07:43:26.327811 1 tasks_processing.go:71] worker 6 working on service_accounts task. I0420 07:43:26.324082 1 tasks_processing.go:71] worker 12 working on ceph_cluster task. I0420 07:43:26.327892 1 gather.go:177] gatherer "clusterconfig" function "node_features" took 4.366828ms to process 0 records I0420 07:43:26.330025 1 tasks_processing.go:71] worker 10 working on cluster_apiserver task. I0420 07:43:26.330062 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 6.370017ms to process 0 records I0420 07:43:26.330069 1 controller.go:129] Initializing last reported time to 0001-01-01T00:00:00Z E0420 07:43:26.330078 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0420 07:43:26.330083 1 controller.go:368] The initial operator extension status is healthy I0420 07:43:26.330090 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 5.579605ms to process 0 records I0420 07:43:26.330092 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0420 07:43:26.330096 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0420 07:43:26.330099 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0420 07:43:26.330115 1 controller.go:531] The operator is still being initialized I0420 07:43:26.330123 1 controller.go:554] The operator is healthy I0420 07:43:26.330181 1 tasks_processing.go:74] worker 42 stopped. I0420 07:43:26.330193 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0420 07:43:26.330198 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0420 07:43:26.330336 1 operator.go:328] started I0420 07:43:26.330361 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0420 07:43:26.330542 1 tasks_processing.go:74] worker 52 stopped. I0420 07:43:26.330560 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 5.713158ms to process 0 records E0420 07:43:26.331446 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.241.15:9091: connect: connection refused I0420 07:43:26.331468 1 tasks_processing.go:74] worker 41 stopped. E0420 07:43:26.331478 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.241.15:9091: connect: connection refused I0420 07:43:26.331503 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 6.463328ms to process 0 records E0420 07:43:26.331508 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.241.15:9091: connect: connection refused E0420 07:43:26.331528 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.241.15:9091: connect: connection refused I0420 07:43:26.331543 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 5.719215ms to process 0 records I0420 07:43:26.331553 1 tasks_processing.go:74] worker 8 stopped. I0420 07:43:26.331743 1 tasks_processing.go:74] worker 60 stopped. I0420 07:43:26.331759 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 7.819776ms to process 0 records I0420 07:43:26.332081 1 tasks_processing.go:74] worker 35 stopped. I0420 07:43:26.332093 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 7.523298ms to process 0 records I0420 07:43:26.332627 1 tasks_processing.go:74] worker 34 stopped. I0420 07:43:26.332633 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 8.339451ms to process 0 records I0420 07:43:26.332639 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 6.988232ms to process 0 records I0420 07:43:26.332646 1 tasks_processing.go:74] worker 55 stopped. I0420 07:43:26.332977 1 tasks_processing.go:74] worker 16 stopped. I0420 07:43:26.332989 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 7.28205ms to process 0 records I0420 07:43:26.333107 1 tasks_processing.go:74] worker 37 stopped. E0420 07:43:26.333119 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0420 07:43:26.333127 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 7.702597ms to process 0 records I0420 07:43:26.333812 1 tasks_processing.go:74] worker 3 stopped. I0420 07:43:26.333827 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 6.792613ms to process 0 records I0420 07:43:26.333833 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 9.637206ms to process 0 records I0420 07:43:26.333838 1 tasks_processing.go:74] worker 43 stopped. I0420 07:43:26.334363 1 tasks_processing.go:74] worker 1 stopped. I0420 07:43:26.334612 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=efe105203e80d8b3a8506777d0db0a068072bb2d090c353ded22bf29c5ce5187 I0420 07:43:26.334630 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 10.911678ms to process 1 records I0420 07:43:26.338103 1 tasks_processing.go:74] worker 18 stopped. I0420 07:43:26.338115 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 12.913009ms to process 0 records I0420 07:43:26.353849 1 tasks_processing.go:74] worker 22 stopped. I0420 07:43:26.353861 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 27.817155ms to process 0 records I0420 07:43:26.353890 1 tasks_processing.go:74] worker 9 stopped. I0420 07:43:26.353903 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 30.324876ms to process 0 records I0420 07:43:26.353966 1 tasks_processing.go:74] worker 48 stopped. I0420 07:43:26.353980 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 27.330771ms to process 0 records I0420 07:43:26.354619 1 tasks_processing.go:74] worker 13 stopped. I0420 07:43:26.355050 1 recorder.go:75] Recording config/network with fingerprint=7db14b4bdafcb3abc48377aed33303cb193748650616a4a1b7b8a15de40aeb34 I0420 07:43:26.355071 1 gather.go:177] gatherer "clusterconfig" function "networks" took 29.652819ms to process 1 records I0420 07:43:26.357786 1 tasks_processing.go:74] worker 4 stopped. I0420 07:43:26.357799 1 gather_logs.go:145] no pods in namespace were found I0420 07:43:26.357803 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 34.249119ms to process 0 records I0420 07:43:26.357823 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 30.566282ms to process 0 records I0420 07:43:26.357831 1 tasks_processing.go:74] worker 25 stopped. I0420 07:43:26.358017 1 tasks_processing.go:74] worker 14 stopped. I0420 07:43:26.358665 1 recorder.go:75] Recording config/infrastructure with fingerprint=732fbfa148d79c20653c2d972a75fa59980a99d39306c6d01aacc8a3dc4db111 I0420 07:43:26.358686 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 31.687602ms to process 1 records I0420 07:43:26.358756 1 recorder.go:75] Recording config/proxy with fingerprint=cdb91ca736e60b7c1f23e8f2192343245b4275b55f64d893d8a48f4d2dab45b9 I0420 07:43:26.358769 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 34.339623ms to process 1 records I0420 07:43:26.358777 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 34.994139ms to process 0 records I0420 07:43:26.358782 1 tasks_processing.go:74] worker 46 stopped. I0420 07:43:26.358819 1 tasks_processing.go:74] worker 11 stopped. I0420 07:43:26.358888 1 tasks_processing.go:74] worker 29 stopped. I0420 07:43:26.358932 1 recorder.go:75] Recording config/authentication with fingerprint=a8d9cbfc70e276c0fe5519e87cbde0e6f6151b9de6617c8867276bd9c47deebb I0420 07:43:26.358944 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 34.553823ms to process 1 records I0420 07:43:26.358953 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 32.202421ms to process 0 records I0420 07:43:26.358959 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 35.283699ms to process 0 records I0420 07:43:26.358963 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 31.250418ms to process 0 records I0420 07:43:26.358968 1 tasks_processing.go:74] worker 27 stopped. I0420 07:43:26.358969 1 tasks_processing.go:74] worker 54 stopped. I0420 07:43:26.358974 1 tasks_processing.go:74] worker 2 stopped. I0420 07:43:26.358987 1 tasks_processing.go:74] worker 21 stopped. E0420 07:43:26.359000 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0420 07:43:26.359013 1 gather.go:177] gatherer "clusterconfig" function "machines" took 32.61265ms to process 0 records I0420 07:43:26.359155 1 tasks_processing.go:74] worker 24 stopped. I0420 07:43:26.359266 1 recorder.go:75] Recording config/ingress with fingerprint=4163f4e26200341b0e445d77b9633739e5e526d5a7cc6fc5b2f2572e2fdf3ced I0420 07:43:26.359278 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 32.149673ms to process 1 records W0420 07:43:26.361379 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0420 07:43:26.362411 1 tasks_processing.go:74] worker 33 stopped. I0420 07:43:26.362509 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=121d84d32d964616efb8525c4db60df7e312e57f47750c443aef93cbc715d2da I0420 07:43:26.362530 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=62e4f7c84b1dbcfa394cc2093490a3739f19a8bf02289a82cdb6c696f5d8d66e I0420 07:43:26.362546 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=f487341a14c414a55da4eb75916efcea607f396c6cc555db5ca71df14eac496c I0420 07:43:26.362562 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=cb9ec7445a2b8f7bcb634f982dfacb064fedd3ad4e8ad58d11d4697919183528 I0420 07:43:26.362579 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=39e6af14e42a95182965d525964775534668613621fffc23b14cd74a169b4484 I0420 07:43:26.362595 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=58592eb295f2325fc350ef556da5997cca712a9d8093554cd73936692922edb5 I0420 07:43:26.362619 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=0941924285a8232f8c28ffaf7da279b00fe94255c78b14565dcd312ea5f36a0a I0420 07:43:26.362639 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=47981b96fd8b2da1c093c842c1771036930e5ab386b51c29c02c54d48c13de08 I0420 07:43:26.362653 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=a96a64cb61191fac30213d812ee68e6b660bab7784e3c4af04051698c72e5d7b I0420 07:43:26.362669 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=83d1a3cd0dcf3aa733ea6a9b0bf9b2c03a79671451d290e934fc662ec36bdbb3 I0420 07:43:26.362685 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=6eca42b3b7119892ffd57e90f78c2310c80f0ea8ba85c3b04613d14268cd6a30 I0420 07:43:26.362691 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 36.855698ms to process 11 records I0420 07:43:26.363170 1 tasks_processing.go:74] worker 61 stopped. I0420 07:43:26.363188 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 36.762863ms to process 0 records I0420 07:43:26.363203 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 36.617977ms to process 0 records I0420 07:43:26.363211 1 tasks_processing.go:74] worker 39 stopped. I0420 07:43:26.363275 1 tasks_processing.go:74] worker 12 stopped. I0420 07:43:26.363287 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 35.346395ms to process 0 records I0420 07:43:26.363352 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=706d66ed2c57b75ece325f8be06d64f951640f8bac3e374c8e7bb7df9f87a33a I0420 07:43:26.363360 1 tasks_processing.go:74] worker 30 stopped. I0420 07:43:26.363363 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 35.650746ms to process 1 records I0420 07:43:26.363395 1 controller.go:254] Source scaController *sca.Controller is not ready I0420 07:43:26.363406 1 controller.go:254] Source clusterTransferController *clustertransfer.Controller is not ready I0420 07:43:26.363410 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0420 07:43:26.363412 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0420 07:43:26.363415 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0420 07:43:26.363431 1 controller.go:531] The operator is still being initialized I0420 07:43:26.363439 1 controller.go:554] The operator is healthy E0420 07:43:26.363990 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0420 07:43:26.363983 1 tasks_processing.go:74] worker 58 stopped. I0420 07:43:26.365793 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 38.876491ms to process 0 records I0420 07:43:26.365815 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 37.810527ms to process 0 records I0420 07:43:26.366681 1 tasks_processing.go:74] worker 15 stopped. I0420 07:43:26.366722 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=00162497cadf194c450ee6f1b28b9a584e2e0f15072734157a6ecc24068fcdd0 I0420 07:43:26.366739 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 38.265012ms to process 1 records E0420 07:43:26.366759 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0420 07:43:26.366782 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0420 07:43:26.366806 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 41.106688ms to process 1 records I0420 07:43:26.366573 1 tasks_processing.go:74] worker 38 stopped. I0420 07:43:26.366842 1 tasks_processing.go:74] worker 5 stopped. I0420 07:43:26.366917 1 tasks_processing.go:74] worker 26 stopped. I0420 07:43:26.367334 1 recorder.go:75] Recording config/oauth with fingerprint=d15373d61379b7aea43f0149813cfa20abfb887f88f4fac067bc41b0a3750e85 I0420 07:43:26.367358 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 40.656838ms to process 1 records I0420 07:43:26.367456 1 tasks_processing.go:74] worker 51 stopped. I0420 07:43:26.367544 1 recorder.go:75] Recording config/image with fingerprint=f1eac2d09cde098225dcb2cbd783310ad8b43043c0bbda6a5da996401ff79536 I0420 07:43:26.368140 1 gather.go:177] gatherer "clusterconfig" function "image" took 39.88255ms to process 1 records I0420 07:43:26.368229 1 tasks_processing.go:74] worker 45 stopped. I0420 07:43:26.369072 1 recorder.go:75] Recording config/node/ip-10-0-0-123.ec2.internal with fingerprint=0984e6cc99e740a6c86e58fbef687c2c5a856baaf339ebdbd12c5665c57bc002 I0420 07:43:26.369139 1 recorder.go:75] Recording config/node/ip-10-0-1-48.ec2.internal with fingerprint=4ec2a725f09deabc96209ae6686d68b43767ea4c0106b05e286fe3fcf7ce000c I0420 07:43:26.369193 1 recorder.go:75] Recording config/node/ip-10-0-2-209.ec2.internal with fingerprint=1c67ed0de7b973ca23ccf1ad333e3d76dc6da5aff2ff5eaee5e3c66ee470f6ce I0420 07:43:26.369201 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 39.94018ms to process 3 records I0420 07:43:26.369297 1 tasks_processing.go:74] worker 10 stopped. I0420 07:43:26.369320 1 recorder.go:75] Recording config/apiserver with fingerprint=1a2bf90d2a3e897c97dce6722a65a27b272fe806aa2ff8604011edb329dcc628 I0420 07:43:26.369326 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 37.191167ms to process 1 records I0420 07:43:26.369426 1 tasks_processing.go:74] worker 7 stopped. I0420 07:43:26.369451 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=3111c4547c6f46aad1e5d2ca7699a6a8c43b17756d56c08c770461779d726634 I0420 07:43:26.369476 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=1625df2ae8607adc2c2d0e51ee3dd5597421b36d2a45057c139b54f2f65a191b I0420 07:43:26.369501 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=dfd92ff8380eb6aec484d044351923708fdc8077ea987e5ce4ed7a6726d1a273 I0420 07:43:26.369540 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 40.743391ms to process 3 records I0420 07:43:26.369604 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=d23f77e7fbf8e021965258e9b1c3b21243ebcf1f81fb47cd9ca27df504b04ce2 I0420 07:43:26.369627 1 tasks_processing.go:74] worker 47 stopped. I0420 07:43:26.369650 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=fade33dc60415c2dca164681220427f74afcb9ad6461b6f1a7331b5963b9cddc I0420 07:43:26.369727 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=2374f52786720f11879fec1121b0e39a351276ec9336b2176f6b3d817f031fc1 I0420 07:43:26.369750 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=a09e95696593b7dfe15f17acf0409cc968d248b2bb898acf739469e4dd4c29d8 I0420 07:43:26.369778 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=8a839193df9678e18d50096083dfba4111d61c45618dd7818552c1a9b8a7d848 I0420 07:43:26.369803 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=028338bfca31e034674f0e59e965b2c8626a4177b2979046abc8f0b2077eb8ef I0420 07:43:26.369825 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=70b3aee7ffe65ecb844874059459e09b49d02018b8a4e38a1cbbc9d8e4308275 I0420 07:43:26.369848 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=c001c0ff8e7eff75c6aa06dc99a8a52d33680485222dab446c6a8cc0336774ff I0420 07:43:26.369871 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=f8336b1612576376a2882c0e5beeec8ac6773292817e2990e54168301dcd40d9 I0420 07:43:26.369902 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=4d478c2d5fb8db1974f11c01f076ffb608d7e43ff53ffeae4e2296242e22de47 I0420 07:43:26.369924 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=fb914603ca7bea70a003dbb989c748764d1e6648a5ffb008538841139efda46b I0420 07:43:26.369947 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=5f098de61b9e26f2fcd129884ecc0e289669bcabbe379dfe42e07c85b54a012e I0420 07:43:26.369969 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=c54308b7f4343ef3f78d08c0ca28dbd566e5147d3761987086679e124a70280d I0420 07:43:26.369989 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 43.600688ms to process 13 records I0420 07:43:26.370078 1 tasks_processing.go:74] worker 40 stopped. I0420 07:43:26.370099 1 recorder.go:75] Recording config/featuregate with fingerprint=25015f7afd5f7d1ab1a865809a9a080f99dd9064e137c847128a87896ef9ae36 I0420 07:43:26.370108 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 43.769137ms to process 1 records I0420 07:43:26.370123 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0420 07:43:26.370130 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 42.11655ms to process 1 records I0420 07:43:26.370132 1 tasks_processing.go:74] worker 59 stopped. I0420 07:43:26.370265 1 tasks_processing.go:74] worker 44 stopped. I0420 07:43:26.370279 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 46.113693ms to process 0 records I0420 07:43:26.370573 1 tasks_processing.go:74] worker 19 stopped. I0420 07:43:26.370589 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 43.269877ms to process 0 records I0420 07:43:26.373140 1 tasks_processing.go:74] worker 0 stopped. I0420 07:43:26.373155 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 49.238081ms to process 0 records I0420 07:43:26.373278 1 tasks_processing.go:74] worker 20 stopped. I0420 07:43:26.373389 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=a8681e79db110ec3afca90d90e0e167580009a2774ed609867cb04b205915150 I0420 07:43:26.373414 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=7287e8efdff121182128582167ef659dae26134d8748d2f00affb33a7ad55137 I0420 07:43:26.373427 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 47.071463ms to process 2 records I0420 07:43:26.379728 1 tasks_processing.go:74] worker 28 stopped. I0420 07:43:26.379820 1 recorder.go:75] Recording config/olm_operators with fingerprint=929a78f7720681718e303887542082bd1aeba857d6dd46374c1a69b0db2a09f5 I0420 07:43:26.379838 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 56.085626ms to process 1 records I0420 07:43:26.379860 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0420 07:43:26.379937 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0420 07:43:26.379957 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0420 07:43:26.383537 1 tasks_processing.go:74] worker 31 stopped. I0420 07:43:26.384297 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=36a71f8668191835605d5b7dd4d598a3e8430730c9b56f0d9eacff041372b1a3 I0420 07:43:26.384544 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=32396d7513aeaf1eda044809b725a10f8b2bddc9c7007353e3de8236ee8a8f69 I0420 07:43:26.384558 1 gather.go:177] gatherer "clusterconfig" function "crds" took 56.359101ms to process 2 records I0420 07:43:26.384834 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0420 07:43:26.385165 1 tasks_processing.go:74] worker 50 stopped. I0420 07:43:26.385396 1 recorder.go:75] Recording config/version with fingerprint=1819e4c05f8cd377e81b0712c4a80f2ceca2f725eb6ec7535154c3aced1f7484 I0420 07:43:26.385412 1 recorder.go:75] Recording config/id with fingerprint=9bfac2ed787cce79a15a4662b44807667e4446c2167b9c6a4331592225e81612 I0420 07:43:26.385421 1 gather.go:177] gatherer "clusterconfig" function "version" took 57.606746ms to process 2 records I0420 07:43:26.385535 1 tasks_processing.go:74] worker 36 stopped. I0420 07:43:26.385587 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=852c745d845720f88ab046e4652eb0d6e05da1b498006ed117a921893fc3e8f8 I0420 07:43:26.385682 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=185a7428009c9e3c77262fac189ff2a07f0434bfde809d819117d300149f1ef5 I0420 07:43:26.385693 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 57.598579ms to process 2 records I0420 07:43:26.392080 1 base_controller.go:82] Caches are synced for ConfigController I0420 07:43:26.392105 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0420 07:43:26.399446 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 07:43:26.400416 1 tasks_processing.go:74] worker 62 stopped. I0420 07:43:26.400485 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 74.922553ms to process 0 records I0420 07:43:26.401142 1 tasks_processing.go:74] worker 23 stopped. I0420 07:43:26.402381 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-66dbbb6468-zcf6m with fingerprint=e887ba8973b42f1418688830da4b59bb850786c33d4ad5b5eae81f493885acc6 I0420 07:43:26.402467 1 recorder.go:75] Recording config/running_containers with fingerprint=3ec5835e446066ab0cda9fa2ce4fe70a91015066835bb5fe099ea3867db9d990 I0420 07:43:26.402476 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 74.912415ms to process 2 records I0420 07:43:26.405960 1 tasks_processing.go:74] worker 63 stopped. E0420 07:43:26.405972 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0420 07:43:26.405982 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2ppoejhtt6juf01aeuheisimfjemt88i-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2ppoejhtt6juf01aeuheisimfjemt88i-primary-cert-bundle-secret" not found I0420 07:43:26.406063 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=3253e57b2482d4be4aba586ea85c2718a9743519d04890a1af3d7c6f7f6a4215 I0420 07:43:26.406076 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 81.937978ms to process 1 records I0420 07:43:26.410812 1 tasks_processing.go:74] worker 56 stopped. E0420 07:43:26.410825 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0420 07:43:26.410830 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0420 07:43:26.410834 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0420 07:43:26.410843 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=ad286723f58bdcfc37aeba3ec5b4110c08e1af59cd34d14b4bfaab02d18e4856 I0420 07:43:26.410891 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0420 07:43:26.410899 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=2ab5ab7b1b10d7fcf1197bb24dea7c90f400e4effc18c7356873209d54fdf84b I0420 07:43:26.410903 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0420 07:43:26.410913 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0420 07:43:26.410950 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0420 07:43:26.410957 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0420 07:43:26.410962 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 84.358451ms to process 7 records I0420 07:43:26.430689 1 base_controller.go:82] Caches are synced for LoggingSyncer I0420 07:43:26.430702 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0420 07:43:26.443305 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0420 07:43:26.448780 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0420 07:43:26.516008 1 cluster_transfer.go:102] no available accepted cluster transfer I0420 07:43:26.516028 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0420 07:43:26.570170 1 sca.go:266] etc-pki-entitlement secret successfully created I0420 07:43:26.570188 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated W0420 07:43:27.361325 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0420 07:43:27.408050 1 tasks_processing.go:74] worker 53 stopped. I0420 07:43:27.408090 1 recorder.go:75] Recording config/alerts with fingerprint=fa0d056dec4cb237605fd2bfbbe04715bf4dba2917237faf58867d82e9434a0e I0420 07:43:27.408105 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 1.083146697s to process 1 records I0420 07:43:27.408113 1 recorder.go:75] Recording config/silenced_alerts.json with fingerprint=37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570 I0420 07:43:27.408119 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 1.082874536s to process 1 records I0420 07:43:27.408122 1 tasks_processing.go:74] worker 57 stopped. W0420 07:43:28.361177 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0420 07:43:28.796390 1 tasks_processing.go:74] worker 17 stopped. I0420 07:43:28.796507 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=2a9577c5e468a2a8d1dcba67bd0f7bfad81c1c4f84e8c51644b0e7bb29dafae5 I0420 07:43:28.796553 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0420 07:43:28.796588 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=2412ad6ec71b738f50ad802d0198db76a2cfcbbe1e1c52647eaded77f30da48e I0420 07:43:28.796598 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0420 07:43:28.796619 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=a21cf05cb9868c754f5aedd04044c59140a5b20603a8e4b04d1a6c66f09f5cde I0420 07:43:28.796635 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0420 07:43:28.796657 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=0f74dcc00797710b64d71670cf6cdf9ad6466f223737e87f659d48aba78a66fa I0420 07:43:28.796683 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=0fffd40c711a7188d889d1ebf25f2c61ab91e3a6248c392640b6aa8ff796c2c7 I0420 07:43:28.796707 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=bd896a1e9930021b26860e7842505d87397607f3e18e26cb363454edf0499284 I0420 07:43:28.796739 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=dec84e9f0686aa4bf8069756df8045454091593f318014093b704bb681ffa460 I0420 07:43:28.797024 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=4d71fc0f4a34d6416758f60e77ec0f04690629ae737b084d8ba165115b69d86c I0420 07:43:28.797063 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=8bb5c306fb524107fff47a81a858b53e45df6af345c867de5aad2b5c6c61de22 I0420 07:43:28.797073 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0420 07:43:28.797090 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=69c22cb700f9fd89bd64b0b27a1b341b4da4cae828c12498dfde2a7b4fbef1fd I0420 07:43:28.797098 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0420 07:43:28.797114 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=2ec7869a20396bcc418b7af1ce35429ae0788ba4d6744d5cd606394587d0a9e0 I0420 07:43:28.797124 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0420 07:43:28.797144 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=b5c7bd3c5e8b9bd69258945e8c050612dc588b35bd9559980979ae9469d652e6 I0420 07:43:28.797154 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0420 07:43:28.797174 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=c3858ff3ad205ebca3ef0b04030ec97a3c1833ebeb1cc44296fd508aa4b73a13 I0420 07:43:28.797291 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=651b69abb9bb901548f10399b9dceaf94fd01305575148953be545af3798e18a I0420 07:43:28.797300 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0420 07:43:28.797307 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0420 07:43:28.797331 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0420 07:43:28.797354 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=305154e787db01c6a483ae20603befbfdf992557360f261a5d3445b902902674 I0420 07:43:28.797376 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=a96466c0e9775178f4de041adabe411713981ebab58b182b3d77d0dc964ffd5f I0420 07:43:28.797385 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0420 07:43:28.797401 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=b94e37f41f87d85ffc4252426125c72335801dc605049cf757f2648916416968 I0420 07:43:28.797409 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0420 07:43:28.797426 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=915d94a65800a37430b9d89b6654387aeef6f4e863c0d55c08a9866ffe7b3a5b I0420 07:43:28.797436 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0420 07:43:28.797448 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=f354ec418c29dd5274244583c4b18062b773136de88900271f1ea97252ad825d I0420 07:43:28.797461 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=e62981aa2b6d36e6e34a3a06009a3fc851d1ef7bbf00dc2ba14aaac98dab0b31 I0420 07:43:28.797477 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=be293f1e72a54e3e578a610abf6d2beee8fc0d936a3628f1f1a165c0b1021630 I0420 07:43:28.797499 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=83a731a28fe6d105ac891f6671077a9f120367463f34555ef7cd3c34aa95460a I0420 07:43:28.797509 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0420 07:43:28.797531 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=da1c6773b07dfd92737596d064a20e631ecd5d08876cbb78f0c1e678cc83463d I0420 07:43:28.797548 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0420 07:43:28.797556 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0420 07:43:28.797564 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.469056813s to process 39 records W0420 07:43:29.361989 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0420 07:43:30.361604 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0420 07:43:31.359282 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0420 07:43:31.359320 1 tasks_processing.go:74] worker 49 stopped. E0420 07:43:31.359331 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0420 07:43:31.359343 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0420 07:43:31.359354 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.033870019s to process 1 records I0420 07:43:39.176052 1 tasks_processing.go:74] worker 32 stopped. I0420 07:43:39.176102 1 recorder.go:75] Recording config/installplans with fingerprint=f17dbfacc3bfddf27ca3b213b39495434cd4c4e9e3dbd69566ffb3845bbcf539 I0420 07:43:39.176115 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.848569462s to process 1 records I0420 07:43:39.936313 1 tasks_processing.go:74] worker 6 stopped. I0420 07:43:39.936612 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=16e4c55b4d63cefcfc7800aa646e617d46f0ec5f940c3fd1a629fc5ae985c964 I0420 07:43:39.936631 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.608472222s to process 1 records E0420 07:43:39.936708 1 periodic.go:254] "Unhandled Error" err="clusterconfig failed after 13.613s with: function \"pod_network_connectivity_checks\" failed with an error, function \"metrics\" failed with an error, function \"tsdb_status\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"machines\" failed with an error, function \"support_secret\" failed with an error, function \"machine_configs\" failed with an error, function \"ingress_certificates\" failed with an error, function \"config_maps\" failed with an error, function \"dvo_metrics\" failed with an error" I0420 07:43:39.937811 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "pod_network_connectivity_checks" failed with an error, function "metrics" failed with an error, function "tsdb_status" failed with an error, function "machine_healthchecks" failed with an error, function "machines" failed with an error, function "support_secret" failed with an error, function "machine_configs" failed with an error, function "ingress_certificates" failed with an error, function "config_maps" failed with an error, function "dvo_metrics" failed with an error I0420 07:43:39.937826 1 periodic.go:216] Running workloads gatherer I0420 07:43:39.937843 1 tasks_processing.go:45] number of workers: 2 I0420 07:43:39.937855 1 tasks_processing.go:69] worker 1 listening for tasks. I0420 07:43:39.937859 1 tasks_processing.go:71] worker 1 working on workload_info task. I0420 07:43:39.937869 1 tasks_processing.go:69] worker 0 listening for tasks. I0420 07:43:39.937890 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0420 07:43:39.945251 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-209.ec2.internal... I0420 07:43:39.945256 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-48.ec2.internal... I0420 07:43:39.945270 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-123.ec2.internal... I0420 07:43:39.974864 1 tasks_processing.go:74] worker 0 stopped. I0420 07:43:39.974881 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 36.9506ms to process 0 records I0420 07:43:44.272299 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 4s I0420 07:43:44.298011 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0420 07:43:44.307907 1 gather_workloads_info.go:387] No image sha256:730d1b6988025bef0daa3a9a5d8467ec4a26b0382cc52f91c3375b4590d3518a (11ms) I0420 07:43:44.318119 1 gather_workloads_info.go:387] No image sha256:a043239802b3eb8b323d285193d2527fad0ecec98ca91d188a3472a2fac8ae04 (10ms) I0420 07:43:44.327598 1 gather_workloads_info.go:387] No image sha256:89277d8d4560d71db88c2dcc67c992a24544ca21810920b609c1d49d53b4a287 (9ms) I0420 07:43:44.336926 1 gather_workloads_info.go:387] No image sha256:5ff204630794311b0b37fa7b197b933ad85d76a481bf7cdb3bcbada08f0cdcbf (9ms) I0420 07:43:44.346239 1 gather_workloads_info.go:387] No image sha256:53ff8efdd8f563421ebc1859ab021a3bbc1306f82d4ff5d9c21bc60ec5f57898 (9ms) I0420 07:43:44.357386 1 gather_workloads_info.go:387] No image sha256:5940cc34b322325618929b1e0eb3864dae73d54c8f5b474e218dfb28123a61c7 (11ms) I0420 07:43:44.366973 1 gather_workloads_info.go:387] No image sha256:9d6d63dce784df5863e6a0ba4c11317a26c9d756d4e62d47759c97190a71cce5 (10ms) I0420 07:43:44.378391 1 gather_workloads_info.go:387] No image sha256:08c5a78c8a5af04c549e2273aaf4bb452a75bf038d68aa9d01bb2aff66c30e90 (11ms) I0420 07:43:44.388257 1 gather_workloads_info.go:387] No image sha256:1eb6a6ee3c66ac15be6bc21bb6015c595c11c15b9dfd21bd9f5ce7f7b990ebaa (10ms) I0420 07:43:44.397992 1 gather_workloads_info.go:387] No image sha256:875c77e5d144f03fb91d8cee0259f6966683ca88d1bf818dbf4652c16b70312c (10ms) I0420 07:43:44.408068 1 gather_workloads_info.go:387] No image sha256:5808401268394502d335281ea1a294b07210461b986b58f91d7d1f29c0029c6d (10ms) I0420 07:43:44.507663 1 gather_workloads_info.go:387] No image sha256:5c6d21c3f97366bc7ab57031cc027b67405a684bd804ce364ed5998b0685eaca (100ms) I0420 07:43:44.608496 1 gather_workloads_info.go:387] No image sha256:5aaea0419169e55832cc27acfe0fe3b9513a343d6bdf71d3da1575ed322245d0 (101ms) I0420 07:43:44.708663 1 gather_workloads_info.go:387] No image sha256:0b2e6536a0ca3afc5e64a218087a6287c74b77a359075a385f1949ca5bd51b22 (100ms) I0420 07:43:44.808525 1 gather_workloads_info.go:387] No image sha256:bdeb1c455cffccdb6c90db238e9d371de06340b1d6105d47f66d87293f878ac1 (100ms) I0420 07:43:44.907396 1 gather_workloads_info.go:387] No image sha256:17aefdc2f4752c23dc8f0947bb01fc34635822eeb18f2d8a5eddf8211b84d450 (99ms) I0420 07:43:45.009287 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (102ms) I0420 07:43:45.108572 1 gather_workloads_info.go:387] No image sha256:52db05084f2a2e7d7bf4040e4865df88ec4074efbbe59a78d861be2c5ab2aa40 (99ms) I0420 07:43:45.214074 1 gather_workloads_info.go:387] No image sha256:084aa9b0f8a6d478549dc384d4e66da13ee9b25cc98531da861cc19dee2a9e8f (105ms) I0420 07:43:45.308563 1 gather_workloads_info.go:387] No image sha256:8f881f6f0498eaba860d711dc8e9dc066db8728f22abedb82e1dc9a8e1c30046 (94ms) I0420 07:43:45.407918 1 gather_workloads_info.go:398] Found image sha256:673ebc8cc22c56c8f410e011b2fa950d28cf7b6420e17fdb6580d6cb10523384 (99ms) I0420 07:43:45.508328 1 gather_workloads_info.go:387] No image sha256:15928a5b803fa00781b408dbc4c3bfa7c5287b18584ba31f817def8a4430c725 (100ms) I0420 07:43:45.609306 1 gather_workloads_info.go:398] Found image sha256:23204877f166455ce8f9de9b91a467eeda576f172d5f96124a68aa986c4775d6 (101ms) I0420 07:43:45.708165 1 gather_workloads_info.go:387] No image sha256:637d41f067a5239096fc22b135181cda5113da833f1370e7a73965e83792e93a (99ms) I0420 07:43:45.808023 1 gather_workloads_info.go:387] No image sha256:0a4dfb8d4c1b3849319d45b4c54dff26a7238a2c08fcaa121f93073e95ab12e8 (100ms) I0420 07:43:45.908091 1 gather_workloads_info.go:387] No image sha256:6814f5fced0ef219d06374011c68a11a3da788a764a00a69ade435466d9ee240 (100ms) I0420 07:43:46.010357 1 gather_workloads_info.go:387] No image sha256:55b1db6038c5beaed54c626e3343b7a8589cc0be8dc41d1a66b4deab766ff520 (102ms) I0420 07:43:46.108226 1 gather_workloads_info.go:387] No image sha256:3bba1358d4a0ae878ff491c0c2cbfffe60649e110b40342b878fe8fa332f8858 (98ms) I0420 07:43:46.210164 1 gather_workloads_info.go:387] No image sha256:0acfe677ed58031f8381cb8a4f2d8917ec4cfa95485e2a46529dffae54c75ace (102ms) I0420 07:43:46.309215 1 gather_workloads_info.go:387] No image sha256:48e883a3932aea9457f55cd4628d21397d429d4828f42f95d1c903d9d5395bde (99ms) I0420 07:43:46.408684 1 gather_workloads_info.go:387] No image sha256:3380fecebaeee139c1607fbcd9346cebc92806679568e2244a8905ef0f52aad2 (99ms) I0420 07:43:46.507844 1 gather_workloads_info.go:387] No image sha256:000105ef5150e7079b90a613fb9e6193e2a6ef9b1908d2dce44f2395d4fd070f (99ms) I0420 07:43:46.607770 1 gather_workloads_info.go:387] No image sha256:15e1b5f763553e86d0230a751b0783e3b990438fd07cf96eed164bccb72ed564 (100ms) I0420 07:43:46.707693 1 gather_workloads_info.go:387] No image sha256:9ef6dc311b880a6442b7a3a19006ddd955c7da36850bf000f5c9b68fd0862dd4 (100ms) I0420 07:43:46.808026 1 gather_workloads_info.go:387] No image sha256:d64bea34bf3e1bb0b3a701c3ff14e66665afc1b050f28124ad7e6888eaec3a81 (100ms) I0420 07:43:46.908271 1 gather_workloads_info.go:387] No image sha256:ab60623bb32f7e75fca71ef65137731cae347a21c7a4091dfd583fa00732721c (100ms) I0420 07:43:47.008566 1 gather_workloads_info.go:387] No image sha256:48f5559439bea030a6423652da448c25fff0c62ba293eb521843c1a8eb89ddb0 (100ms) I0420 07:43:47.109069 1 gather_workloads_info.go:387] No image sha256:15677f0b70e6aa2dfaf088e45fc1a425c22bf6fda326b8116f87e88b6694dfab (100ms) I0420 07:43:47.208050 1 gather_workloads_info.go:387] No image sha256:6c6d412ccf0c3d6a14645a79df6c4658c0f8725d7954cb83c3d2f0116499345a (99ms) I0420 07:43:47.308163 1 gather_workloads_info.go:387] No image sha256:25c148fd380b1a9db3f6039d2e0eabc489a954921452391390ba9192b2325678 (100ms) I0420 07:43:47.409002 1 gather_workloads_info.go:387] No image sha256:934d8e8c50f3c609b8eea80d1051111fe3d066fe8c65c79572072ae55fcb0a86 (101ms) I0420 07:43:47.508432 1 gather_workloads_info.go:387] No image sha256:572ebf7c7fab1a518582f3ca46c866730423cce6e8a19f614dc15fd61bf0c65a (99ms) I0420 07:43:47.608552 1 gather_workloads_info.go:387] No image sha256:5ac9c549d65fc1d8bc900773bebee43e9192bcec1bb5fa46afb4597230c16ac7 (100ms) I0420 07:43:47.708554 1 gather_workloads_info.go:387] No image sha256:30e597ec5d6bb96ff70a4f8688c748b659cd4fd5d73d222e8701821d236795c5 (100ms) I0420 07:43:47.807942 1 gather_workloads_info.go:387] No image sha256:5b6212b8f539f08e78417d8a4b7485ca0b4e7927cacd7b752742a28841bc8ccd (99ms) I0420 07:43:47.908203 1 gather_workloads_info.go:387] No image sha256:5d6a2b8d2508c7da71192661e7c0dcf1335f48b6336a20189abffbe79eeff962 (100ms) I0420 07:43:47.908237 1 tasks_processing.go:74] worker 1 stopped. I0420 07:43:47.908562 1 recorder.go:75] Recording config/workload_info with fingerprint=e6caff31ad277f391030b3f0f73b7b5831ad45821a39c30d0d2069842487b2a8 I0420 07:43:47.908579 1 gather.go:177] gatherer "workloads" function "workload_info" took 7.970370326s to process 1 records I0420 07:43:47.908591 1 periodic.go:263] Periodic gather workloads completed in 7.97s I0420 07:43:47.908599 1 controllerstatus.go:80] name=periodic-workloads healthy=true reason= message= I0420 07:43:47.908603 1 periodic.go:216] Running conditional gatherer I0420 07:43:47.915586 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.21.9/gathering_rules I0420 07:43:48.515468 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer I0420 07:43:49.559294 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0420 07:43:49.566297 1 conditional_gatherer.go:392] cluster version is '4.21.9' I0420 07:43:49.566325 1 tasks_processing.go:45] number of workers: 3 I0420 07:43:49.566332 1 tasks_processing.go:69] worker 2 listening for tasks. I0420 07:43:49.566336 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0420 07:43:49.566346 1 tasks_processing.go:69] worker 0 listening for tasks. I0420 07:43:49.566364 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0420 07:43:49.566367 1 tasks_processing.go:69] worker 1 listening for tasks. I0420 07:43:49.566384 1 tasks_processing.go:74] worker 1 stopped. I0420 07:43:49.566383 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0420 07:43:49.566418 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=c8d4892cd4f2eea27d57353c6055eeb530bc2d811bfe041f6984d90395255a8c I0420 07:43:49.566433 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 857ns to process 1 records I0420 07:43:49.566467 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0420 07:43:49.566475 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.265µs to process 1 records I0420 07:43:49.566480 1 tasks_processing.go:74] worker 0 stopped. I0420 07:43:49.566647 1 tasks_processing.go:74] worker 2 stopped. I0420 07:43:49.566662 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 245.937µs to process 0 records I0420 07:43:49.566689 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0420 07:43:49.566717 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0420 07:43:49.596844 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=f7dc08d1ea26be81f451f421b0a87a0afa88c1c4d2883447ddbc5e6ae8c15cfa I0420 07:43:49.597015 1 diskrecorder.go:70] Writing 111 records to /var/lib/insights-operator/insights-2026-04-20-074349.tar.gz I0420 07:43:49.603838 1 diskrecorder.go:51] Wrote 111 records to disk in 6ms I0420 07:43:49.603873 1 periodic.go:285] Gathering cluster info every 2h0m0s I0420 07:43:49.603894 1 periodic.go:286] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0420 07:44:56.323833 1 diskrecorder.go:223] Found files to send: insights-2026-04-20-074349.tar.gz I0420 07:44:56.323863 1 insightsuploader.go:150] Checking archives to upload periodically every 15m7.203763065s I0420 07:44:56.323872 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0420 07:44:56.337433 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0420 07:44:56.666142 1 requests.go:87] Successfully reported id=2026-04-20T07:44:56Z x-rh-insights-request-id=8dceb2a4e1fe42e89053b2854764459c, wrote=48144 I0420 07:44:56.666262 1 insightsuploader.go:187] Uploaded report successfully in 342.384058ms I0420 07:44:56.666282 1 controller.go:129] Initializing last reported time to 2026-04-20T07:44:56Z I0420 07:44:56.666300 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0420 07:44:56.666315 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0420 07:44:56.666325 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0420 07:44:56.673132 1 controller.go:554] The operator is healthy I0420 07:45:26.338931 1 controller.go:554] The operator is healthy I0420 07:46:00.866472 1 insightsreport.go:137] Pulling report from smart-proxy I0420 07:46:00.866501 1 insightsreport.go:149] Retrieving report I0420 07:46:00.874426 1 requests.go:110] Retrieving report for cluster: 862829eb-adf4-48f4-a086-2fd00f92d489 I0420 07:46:00.874440 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/862829eb-adf4-48f4-a086-2fd00f92d489/reports I0420 07:46:00.877582 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/862829eb-adf4-48f4-a086-2fd00f92d489/reports I0420 07:46:01.063833 1 insightsreport.go:184] Report retrieved I0420 07:46:01.092512 1 insightsreport.go:239] Report retrieved correctly I0420 07:47:26.340515 1 controller.go:554] The operator is healthy I0420 07:48:26.314806 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 07:48:26.325693 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 07:48:26.325710 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 07:48:26.331809 1 secretconfigobserver.go:119] support secret does not exist I0420 07:49:26.337444 1 controller.go:554] The operator is healthy I0420 07:51:26.338787 1 controller.go:554] The operator is healthy I0420 07:53:26.309199 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 07:53:26.332413 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 07:53:26.337314 1 controller.go:554] The operator is healthy I0420 07:53:26.338098 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 07:53:26.338117 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 07:53:26.343140 1 secretconfigobserver.go:119] support secret does not exist I0420 07:55:26.339125 1 controller.go:554] The operator is healthy I0420 07:57:26.338827 1 controller.go:554] The operator is healthy I0420 07:58:26.343470 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 07:58:26.350754 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 07:58:26.350782 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 07:58:26.356832 1 secretconfigobserver.go:119] support secret does not exist I0420 07:59:26.338591 1 controller.go:554] The operator is healthy I0420 08:00:08.871381 1 insightsuploader.go:146] Nothing to report since 2026-04-20T07:44:56Z I0420 08:01:26.338546 1 controller.go:554] The operator is healthy I0420 08:03:26.309683 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 08:03:26.315118 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 08:03:26.338749 1 controller.go:554] The operator is healthy I0420 08:03:26.357623 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 08:03:26.363778 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 08:03:26.363793 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 08:03:26.369504 1 secretconfigobserver.go:119] support secret does not exist I0420 08:05:26.339208 1 controller.go:554] The operator is healthy I0420 08:07:26.339106 1 controller.go:554] The operator is healthy I0420 08:08:26.370580 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 08:08:26.381649 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 08:08:26.381669 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 08:08:26.389148 1 secretconfigobserver.go:119] support secret does not exist I0420 08:09:26.340861 1 controller.go:554] The operator is healthy I0420 08:11:26.339249 1 controller.go:554] The operator is healthy I0420 08:13:26.310539 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 08:13:26.316059 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 08:13:26.337739 1 controller.go:554] The operator is healthy I0420 08:13:26.389955 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 08:13:26.396060 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 08:13:26.396078 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 08:13:26.401651 1 secretconfigobserver.go:119] support secret does not exist I0420 08:15:26.338416 1 controller.go:554] The operator is healthy I0420 08:16:25.320601 1 insightsuploader.go:146] Nothing to report since 2026-04-20T07:44:56Z I0420 08:17:26.337599 1 controller.go:554] The operator is healthy I0420 08:18:26.401809 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 08:18:26.408917 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 08:18:26.408934 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 08:18:26.414427 1 secretconfigobserver.go:119] support secret does not exist I0420 08:19:26.337872 1 controller.go:554] The operator is healthy