I0406 02:57:29.769147 1 cmd.go:253] Using service-serving-cert provided certificates I0406 02:57:29.769546 1 observer_polling.go:159] Starting file observer I0406 02:57:30.268198 1 operator.go:76] Starting insights-operator v0.0.0-master+$Format:%H$ I0406 02:57:30.268406 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0406 02:57:30.268936 1 secure_serving.go:57] Forcing use of http/1.1 only W0406 02:57:30.268955 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0406 02:57:30.268959 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0406 02:57:30.268962 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0406 02:57:30.268965 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0406 02:57:30.268967 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0406 02:57:30.268969 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0406 02:57:30.269035 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0406 02:57:30.272406 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"04902f22-0f63-4d95-a18a-a9ba924307f5", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "HyperShiftOnlyDynamicResourceAllocation", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryption", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLM", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "OpenShiftPodSecurityAdmission", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}} I0406 02:57:30.272400 1 operator.go:141] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNSInstall GatewayAPI GatewayAPIController HighlyAvailableArbiter HyperShiftOnlyDynamicResourceAllocation ImageStreamImportMode ImageVolume KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS ManagedBootImagesAzure ManagedBootImagesvSphere MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages PreconfiguredUDNAddresses ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SigstoreImageVerification SigstoreImageVerificationPKI StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSDualStackInstall AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureDualStackInstall AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement CBORServingAndStorage CRDCompatibilityRequirementOperator ClientsAllowCBOR ClientsPreferCBOR ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterAPIMachineManagement ClusterAPIMachineManagementVSphere ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud EtcdBackendQuota EventTTL EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall GCPDualStackInstall ImageModeStatusReporting IngressControllerDynamicConfigurationManager InsightsConfig InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryption KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesCPMS MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutableCSINodeAllocatableCount MutatingAdmissionPolicy NewOLM NewOLMBoxCutterRuntime NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterInstall NutanixMultiSubnets OSStreams OVNObservability OnPremDNSRecords OpenShiftPodSecurityAdmission ProvisioningRequestAvailable SELinuxMount ShortCertRotation SignatureStores TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeGroupSnapshot] I0406 02:57:30.273924 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0406 02:57:30.274185 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0406 02:57:30.274195 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0406 02:57:30.274189 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0406 02:57:30.274217 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0406 02:57:30.274218 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0406 02:57:30.274233 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0406 02:57:30.274387 1 secure_serving.go:213] Serving securely on [::]:8443 I0406 02:57:30.274421 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0406 02:57:30.277695 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0406 02:57:30.277717 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0406 02:57:30.277769 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0406 02:57:30.281710 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0406 02:57:30.281739 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0406 02:57:30.285322 1 secretconfigobserver.go:119] support secret does not exist I0406 02:57:30.295681 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0406 02:57:30.299571 1 secretconfigobserver.go:119] support secret does not exist I0406 02:57:30.299951 1 recorder.go:176] Pruning old reports every 5h26m32s, max age is 288h0m0s I0406 02:57:30.304718 1 periodic.go:216] Running clusterconfig gatherer I0406 02:57:30.304747 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0406 02:57:30.304759 1 insightsreport.go:296] Starting report retriever I0406 02:57:30.304761 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0406 02:57:30.304764 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0406 02:57:30.304772 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0406 02:57:30.304777 1 tasks_processing.go:45] number of workers: 64 I0406 02:57:30.304795 1 tasks_processing.go:69] worker 4 listening for tasks. I0406 02:57:30.304795 1 tasks_processing.go:69] worker 2 listening for tasks. I0406 02:57:30.304802 1 tasks_processing.go:69] worker 3 listening for tasks. I0406 02:57:30.304796 1 tasks_processing.go:69] worker 1 listening for tasks. I0406 02:57:30.304804 1 tasks_processing.go:69] worker 0 listening for tasks. I0406 02:57:30.304808 1 tasks_processing.go:69] worker 17 listening for tasks. I0406 02:57:30.304810 1 tasks_processing.go:69] worker 10 listening for tasks. I0406 02:57:30.304820 1 tasks_processing.go:69] worker 6 listening for tasks. I0406 02:57:30.304821 1 tasks_processing.go:71] worker 10 working on sap_pods task. I0406 02:57:30.304823 1 tasks_processing.go:69] worker 34 listening for tasks. I0406 02:57:30.304823 1 tasks_processing.go:69] worker 33 listening for tasks. I0406 02:57:30.304828 1 tasks_processing.go:69] worker 56 listening for tasks. I0406 02:57:30.304831 1 tasks_processing.go:69] worker 35 listening for tasks. I0406 02:57:30.304833 1 tasks_processing.go:69] worker 8 listening for tasks. I0406 02:57:30.304822 1 tasks_processing.go:71] worker 17 working on container_images task. I0406 02:57:30.304838 1 tasks_processing.go:69] worker 36 listening for tasks. I0406 02:57:30.304840 1 tasks_processing.go:69] worker 57 listening for tasks. I0406 02:57:30.304839 1 tasks_processing.go:69] worker 15 listening for tasks. I0406 02:57:30.304825 1 tasks_processing.go:69] worker 7 listening for tasks. I0406 02:57:30.304849 1 tasks_processing.go:69] worker 38 listening for tasks. I0406 02:57:30.304848 1 tasks_processing.go:69] worker 14 listening for tasks. I0406 02:57:30.304855 1 tasks_processing.go:69] worker 39 listening for tasks. I0406 02:57:30.304815 1 tasks_processing.go:69] worker 5 listening for tasks. I0406 02:57:30.304853 1 tasks_processing.go:69] worker 9 listening for tasks. I0406 02:57:30.304864 1 tasks_processing.go:69] worker 40 listening for tasks. I0406 02:57:30.304862 1 tasks_processing.go:69] worker 55 listening for tasks. I0406 02:57:30.304865 1 tasks_processing.go:69] worker 53 listening for tasks. I0406 02:57:30.304869 1 tasks_processing.go:69] worker 41 listening for tasks. I0406 02:57:30.304816 1 tasks_processing.go:69] worker 42 listening for tasks. I0406 02:57:30.304870 1 tasks_processing.go:69] worker 46 listening for tasks. I0406 02:57:30.304878 1 tasks_processing.go:69] worker 59 listening for tasks. I0406 02:57:30.304821 1 tasks_processing.go:69] worker 12 listening for tasks. I0406 02:57:30.304880 1 tasks_processing.go:69] worker 47 listening for tasks. I0406 02:57:30.304882 1 tasks_processing.go:69] worker 31 listening for tasks. I0406 02:57:30.304880 1 tasks_processing.go:69] worker 63 listening for tasks. I0406 02:57:30.304888 1 tasks_processing.go:69] worker 26 listening for tasks. I0406 02:57:30.304891 1 tasks_processing.go:69] worker 24 listening for tasks. I0406 02:57:30.304893 1 tasks_processing.go:69] worker 62 listening for tasks. I0406 02:57:30.304837 1 tasks_processing.go:69] worker 44 listening for tasks. I0406 02:57:30.304900 1 tasks_processing.go:69] worker 32 listening for tasks. I0406 02:57:30.304837 1 tasks_processing.go:69] worker 18 listening for tasks. I0406 02:57:30.304841 1 tasks_processing.go:69] worker 45 listening for tasks. I0406 02:57:30.304845 1 tasks_processing.go:69] worker 37 listening for tasks. I0406 02:57:30.304845 1 tasks_processing.go:69] worker 19 listening for tasks. I0406 02:57:30.304907 1 tasks_processing.go:69] worker 49 listening for tasks. I0406 02:57:30.304847 1 tasks_processing.go:69] worker 51 listening for tasks. I0406 02:57:30.304852 1 tasks_processing.go:69] worker 20 listening for tasks. I0406 02:57:30.304854 1 tasks_processing.go:69] worker 52 listening for tasks. I0406 02:57:30.304860 1 tasks_processing.go:69] worker 58 listening for tasks. I0406 02:57:30.304861 1 tasks_processing.go:69] worker 54 listening for tasks. I0406 02:57:30.304861 1 tasks_processing.go:69] worker 21 listening for tasks. I0406 02:57:30.304860 1 tasks_processing.go:69] worker 50 listening for tasks. I0406 02:57:30.304864 1 tasks_processing.go:71] worker 4 working on cluster_apiserver task. I0406 02:57:30.304830 1 tasks_processing.go:69] worker 43 listening for tasks. I0406 02:57:30.304869 1 tasks_processing.go:71] worker 2 working on machine_config_pools task. I0406 02:57:30.304867 1 tasks_processing.go:69] worker 22 listening for tasks. I0406 02:57:30.304873 1 tasks_processing.go:69] worker 48 listening for tasks. I0406 02:57:30.304816 1 tasks_processing.go:69] worker 11 listening for tasks. I0406 02:57:30.304874 1 tasks_processing.go:71] worker 3 working on openstack_version task. I0406 02:57:30.304969 1 tasks_processing.go:71] worker 6 working on storage_cluster task. I0406 02:57:30.304976 1 tasks_processing.go:71] worker 11 working on active_alerts task. I0406 02:57:30.304872 1 tasks_processing.go:69] worker 27 listening for tasks. I0406 02:57:30.305027 1 tasks_processing.go:71] worker 47 working on proxies task. I0406 02:57:30.305031 1 tasks_processing.go:71] worker 27 working on nodes task. I0406 02:57:30.305035 1 tasks_processing.go:71] worker 39 working on jaegers task. I0406 02:57:30.305050 1 tasks_processing.go:71] worker 53 working on ingress task. I0406 02:57:30.305065 1 tasks_processing.go:71] worker 46 working on metrics task. I0406 02:57:30.305069 1 tasks_processing.go:71] worker 12 working on nodenetworkconfigurationpolicies task. I0406 02:57:30.304875 1 tasks_processing.go:69] worker 28 listening for tasks. I0406 02:57:30.305080 1 tasks_processing.go:71] worker 28 working on node_logs task. I0406 02:57:30.305087 1 tasks_processing.go:71] worker 41 working on openstack_dataplanedeployments task. I0406 02:57:30.305100 1 tasks_processing.go:71] worker 5 working on node_features task. I0406 02:57:30.305066 1 tasks_processing.go:71] worker 59 working on service_accounts task. I0406 02:57:30.305107 1 tasks_processing.go:71] worker 9 working on operators_pods_and_events task. I0406 02:57:30.304874 1 tasks_processing.go:69] worker 60 listening for tasks. I0406 02:57:30.305356 1 tasks_processing.go:71] worker 55 working on infrastructures task. I0406 02:57:30.305406 1 tasks_processing.go:71] worker 52 working on silenced_alerts task. I0406 02:57:30.304880 1 tasks_processing.go:69] worker 23 listening for tasks. I0406 02:57:30.305511 1 tasks_processing.go:71] worker 23 working on lokistack task. I0406 02:57:30.305521 1 tasks_processing.go:71] worker 22 working on sap_config task. I0406 02:57:30.305633 1 tasks_processing.go:71] worker 48 working on networks task. I0406 02:57:30.305720 1 tasks_processing.go:71] worker 26 working on openshift_logging task. I0406 02:57:30.305755 1 tasks_processing.go:71] worker 58 working on nodenetworkstates task. I0406 02:57:30.305922 1 tasks_processing.go:71] worker 18 working on validating_webhook_configurations task. I0406 02:57:30.306018 1 tasks_processing.go:71] worker 62 working on machine_configs task. I0406 02:57:30.306355 1 tasks_processing.go:71] worker 24 working on mutating_webhook_configurations task. I0406 02:57:30.305512 1 tasks_processing.go:71] worker 31 working on image task. I0406 02:57:30.306482 1 tasks_processing.go:71] worker 36 working on olm_operators task. I0406 02:57:30.306559 1 tasks_processing.go:71] worker 33 working on qemu_kubevirt_launcher_logs task. I0406 02:57:30.306581 1 tasks_processing.go:71] worker 44 working on monitoring_persistent_volumes task. I0406 02:57:30.304882 1 tasks_processing.go:71] worker 0 working on support_secret task. I0406 02:57:30.306705 1 tasks_processing.go:71] worker 32 working on storage_classes task. I0406 02:57:30.306898 1 tasks_processing.go:71] worker 35 working on openshift_machine_api_events task. I0406 02:57:30.304879 1 tasks_processing.go:69] worker 30 listening for tasks. I0406 02:57:30.304886 1 tasks_processing.go:69] worker 25 listening for tasks. I0406 02:57:30.304811 1 tasks_processing.go:69] worker 13 listening for tasks. I0406 02:57:30.304891 1 tasks_processing.go:69] worker 61 listening for tasks. I0406 02:57:30.304892 1 tasks_processing.go:69] worker 16 listening for tasks. I0406 02:57:30.304966 1 tasks_processing.go:71] worker 34 working on pdbs task. I0406 02:57:30.307165 1 tasks_processing.go:71] worker 30 working on container_runtime_configs task. I0406 02:57:30.307206 1 tasks_processing.go:71] worker 7 working on operators task. I0406 02:57:30.307433 1 tasks_processing.go:71] worker 38 working on openstack_dataplanenodesets task. I0406 02:57:30.304878 1 tasks_processing.go:71] worker 1 working on overlapping_namespace_uids task. I0406 02:57:30.307458 1 tasks_processing.go:71] worker 45 working on oauths task. I0406 02:57:30.305381 1 tasks_processing.go:71] worker 51 working on crds task. I0406 02:57:30.307506 1 tasks_processing.go:71] worker 25 working on version task. I0406 02:57:30.307510 1 tasks_processing.go:71] worker 14 working on certificate_signing_requests task. I0406 02:57:30.307586 1 tasks_processing.go:71] worker 57 working on install_plans task. I0406 02:57:30.305111 1 tasks_processing.go:71] worker 54 working on ingress_certificates task. I0406 02:57:30.305116 1 tasks_processing.go:71] worker 20 working on clusterroles task. I0406 02:57:30.305286 1 tasks_processing.go:71] worker 49 working on dvo_metrics task. I0406 02:57:30.306486 1 tasks_processing.go:71] worker 56 working on tsdb_status task. I0406 02:57:30.305307 1 tasks_processing.go:71] worker 42 working on feature_gates task. I0406 02:57:30.305108 1 tasks_processing.go:71] worker 19 working on machine_autoscalers task. I0406 02:57:30.305392 1 tasks_processing.go:71] worker 43 working on machine_sets task. I0406 02:57:30.305390 1 tasks_processing.go:71] worker 60 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0406 02:57:30.305400 1 tasks_processing.go:71] worker 50 working on aggregated_monitoring_cr_names task. I0406 02:57:30.305476 1 tasks_processing.go:71] worker 63 working on image_registries task. I0406 02:57:30.307473 1 tasks_processing.go:71] worker 16 working on schedulers task. I0406 02:57:30.304884 1 tasks_processing.go:69] worker 29 listening for tasks. I0406 02:57:30.307672 1 tasks_processing.go:71] worker 15 working on authentication task. I0406 02:57:30.308660 1 tasks_processing.go:71] worker 8 working on ceph_cluster task. I0406 02:57:30.307677 1 tasks_processing.go:71] worker 13 working on pod_network_connectivity_checks task. I0406 02:57:30.307710 1 tasks_processing.go:71] worker 61 working on config_maps task. I0406 02:57:30.305286 1 tasks_processing.go:71] worker 40 working on machine_healthchecks task. I0406 02:57:30.305396 1 tasks_processing.go:71] worker 21 working on sap_datahubs task. I0406 02:57:30.309243 1 tasks_processing.go:71] worker 2 working on openstack_controlplanes task. I0406 02:57:30.309247 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 4.280696ms to process 0 records I0406 02:57:30.309289 1 tasks_processing.go:71] worker 37 working on cost_management_metrics_configs task. I0406 02:57:30.309311 1 tasks_processing.go:71] worker 58 working on image_pruners task. I0406 02:57:30.309358 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 3.54271ms to process 0 records I0406 02:57:30.309708 1 tasks_processing.go:74] worker 26 stopped. I0406 02:57:30.309723 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 3.956521ms to process 0 records I0406 02:57:30.310042 1 tasks_processing.go:71] worker 29 working on machines task. I0406 02:57:30.310157 1 controller.go:129] Initializing last reported time to 0001-01-01T00:00:00Z I0406 02:57:30.310174 1 controller.go:368] The initial operator extension status is healthy I0406 02:57:30.310182 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0406 02:57:30.310186 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0406 02:57:30.310189 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0406 02:57:30.310202 1 controller.go:531] The operator is still being initialized I0406 02:57:30.310207 1 controller.go:554] The operator is healthy I0406 02:57:30.310278 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0406 02:57:30.310318 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0406 02:57:30.310392 1 operator.go:328] started I0406 02:57:30.310412 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0406 02:57:30.311410 1 tasks_processing.go:74] worker 47 stopped. I0406 02:57:30.311504 1 recorder.go:75] Recording config/proxy with fingerprint=e27ab6a7bd0dd6879285e68d8019c090b5e0e290e1c24c98d241b4fb703391bc I0406 02:57:30.311521 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 6.368292ms to process 1 records E0406 02:57:30.312936 1 gather_silenced_alerts.go:52] Unable to retrieve silenced alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp 172.30.230.250:9094: connect: connection refused I0406 02:57:30.312955 1 tasks_processing.go:74] worker 52 stopped. E0406 02:57:30.312965 1 gather.go:140] gatherer "clusterconfig" function "silenced_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp 172.30.230.250:9094: connect: connection refused I0406 02:57:30.312979 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 7.528726ms to process 0 records E0406 02:57:30.312989 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.60.225:9091: connect: connection refused I0406 02:57:30.313004 1 tasks_processing.go:74] worker 56 stopped. E0406 02:57:30.313010 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.60.225:9091: connect: connection refused E0406 02:57:30.313010 1 gather_active_alerts.go:64] Unable to retrieve most recent alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp 172.30.230.250:9094: connect: connection refused E0406 02:57:30.312988 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.60.225:9091: connect: connection refused I0406 02:57:30.313018 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 5.085315ms to process 0 records E0406 02:57:30.313027 1 gather.go:140] gatherer "clusterconfig" function "active_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp 172.30.230.250:9094: connect: connection refused I0406 02:57:30.313034 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 8.034357ms to process 0 records E0406 02:57:30.313040 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.60.225:9091: connect: connection refused I0406 02:57:30.313055 1 tasks_processing.go:74] worker 11 stopped. I0406 02:57:30.313056 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 7.950802ms to process 0 records I0406 02:57:30.313060 1 tasks_processing.go:74] worker 46 stopped. I0406 02:57:30.314891 1 tasks_processing.go:74] worker 41 stopped. I0406 02:57:30.314904 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 9.791843ms to process 0 records I0406 02:57:30.314911 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 9.818478ms to process 0 records I0406 02:57:30.314917 1 tasks_processing.go:74] worker 12 stopped. I0406 02:57:30.315654 1 tasks_processing.go:74] worker 31 stopped. I0406 02:57:30.315718 1 recorder.go:75] Recording config/image with fingerprint=85d096bbb6133b923bc1528ec6297dd258c0f67ba17fd161628e09f43c68dbfe I0406 02:57:30.315746 1 gather.go:177] gatherer "clusterconfig" function "image" took 9.193195ms to process 1 records I0406 02:57:30.316154 1 tasks_processing.go:74] worker 24 stopped. I0406 02:57:30.316319 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=d47d8e3e52be722e0356e40ae6f9afa330f119092fe1f5e123e500b16f42639b I0406 02:57:30.316348 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=43b0746f5a7620ed9eb834eb273ac4d8070a1aa0e1576e459f33d5b91ab3cb01 I0406 02:57:30.316372 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=1f2348b40f913d2d1cd2fdeed5c6af2ce8a4ab1c3c03f567b154a749803dd3c1 I0406 02:57:30.316380 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 9.77876ms to process 3 records I0406 02:57:30.316709 1 tasks_processing.go:74] worker 0 stopped. E0406 02:57:30.316721 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0406 02:57:30.316745 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 10.054643ms to process 0 records I0406 02:57:30.317268 1 tasks_processing.go:74] worker 53 stopped. I0406 02:57:30.317440 1 recorder.go:75] Recording config/ingress with fingerprint=f7eba242ca281399a28169095a7e14fc5a66a14945293d544b8e4a5af0a6e224 I0406 02:57:30.317455 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 12.205997ms to process 1 records I0406 02:57:30.317957 1 tasks_processing.go:74] worker 38 stopped. I0406 02:57:30.317971 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 10.505355ms to process 0 records I0406 02:57:30.318121 1 tasks_processing.go:74] worker 43 stopped. I0406 02:57:30.318132 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 10.032335ms to process 0 records I0406 02:57:30.318999 1 gather_logs.go:145] no pods in namespace were found I0406 02:57:30.319012 1 tasks_processing.go:74] worker 33 stopped. I0406 02:57:30.319019 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 12.437734ms to process 0 records I0406 02:57:30.319303 1 tasks_processing.go:74] worker 36 stopped. I0406 02:57:30.319316 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 12.791679ms to process 0 records I0406 02:57:30.324371 1 tasks_processing.go:74] worker 29 stopped. E0406 02:57:30.324382 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0406 02:57:30.324389 1 gather.go:177] gatherer "clusterconfig" function "machines" took 14.314509ms to process 0 records I0406 02:57:30.324835 1 tasks_processing.go:74] worker 23 stopped. I0406 02:57:30.324847 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 19.307285ms to process 0 records E0406 02:57:30.325651 1 gather_node_features.go:86] GatherNodeFeatures: NodeFeatures resource not found in openshift-nfd namespace (may not be installed) I0406 02:57:30.325665 1 tasks_processing.go:74] worker 5 stopped. I0406 02:57:30.325674 1 gather.go:177] gatherer "clusterconfig" function "node_features" took 20.554748ms to process 0 records I0406 02:57:30.325813 1 tasks_processing.go:74] worker 10 stopped. I0406 02:57:30.325823 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 20.98649ms to process 0 records I0406 02:57:30.325828 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 17.749653ms to process 0 records I0406 02:57:30.325832 1 tasks_processing.go:74] worker 19 stopped. I0406 02:57:30.333611 1 tasks_processing.go:74] worker 22 stopped. I0406 02:57:30.333624 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 28.082342ms to process 0 records I0406 02:57:30.333652 1 tasks_processing.go:74] worker 32 stopped. I0406 02:57:30.333765 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=65a858ad9446040bbe8808432b664476d0818cd192cd581c1030fc4012c76e05 I0406 02:57:30.333785 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=80ac4b7294a8926d5cbefbd36cc6ba76f0a4df2c69f84530217dc21d785a5f1d I0406 02:57:30.333792 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 26.867623ms to process 2 records I0406 02:57:30.333837 1 tasks_processing.go:74] worker 21 stopped. I0406 02:57:30.333882 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 24.77958ms to process 0 records E0406 02:57:30.333897 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0406 02:57:30.333906 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 24.879163ms to process 0 records I0406 02:57:30.333908 1 tasks_processing.go:74] worker 40 stopped. I0406 02:57:30.333913 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 28.897718ms to process 0 records I0406 02:57:30.333921 1 tasks_processing.go:74] worker 3 stopped. I0406 02:57:30.334020 1 tasks_processing.go:74] worker 55 stopped. I0406 02:57:30.334677 1 recorder.go:75] Recording config/infrastructure with fingerprint=402612e5fc3222a0ac1e1718036f47ad5c60a595fbe5edac93b319c6f2d093d9 I0406 02:57:30.334697 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 28.601132ms to process 1 records I0406 02:57:30.334748 1 tasks_processing.go:74] worker 45 stopped. I0406 02:57:30.335358 1 recorder.go:75] Recording config/oauth with fingerprint=e6b0f66f63b0d9fcdbcfa0bf97e1e18530edf1e2e6c36654687a5d8a65f811b5 I0406 02:57:30.335559 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 26.660171ms to process 1 records I0406 02:57:30.335609 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 29.289952ms to process 0 records I0406 02:57:30.335620 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 25.167862ms to process 0 records I0406 02:57:30.335626 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 25.341636ms to process 0 records I0406 02:57:30.335707 1 tasks_processing.go:74] worker 37 stopped. I0406 02:57:30.335722 1 tasks_processing.go:74] worker 39 stopped. I0406 02:57:30.335745 1 tasks_processing.go:74] worker 2 stopped. I0406 02:57:30.335815 1 tasks_processing.go:74] worker 4 stopped. I0406 02:57:30.335953 1 recorder.go:75] Recording config/apiserver with fingerprint=885a80cd0f3a1aea6f7ca5ea6276c1b49821acbc9c8c2320b630a5af028c039f I0406 02:57:30.335976 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 29.721425ms to process 1 records I0406 02:57:30.336133 1 recorder.go:75] Recording config/featuregate with fingerprint=5734bedb0a16b41d9c43198807c88d984d57e73aaca2223711d9e06f0d8294d6 I0406 02:57:30.336144 1 tasks_processing.go:74] worker 42 stopped. I0406 02:57:30.336145 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 26.815623ms to process 1 records I0406 02:57:30.341460 1 tasks_processing.go:74] worker 18 stopped. I0406 02:57:30.341567 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=8c9ba17b7a92e4f1479673e35de530eea32cac8732d9eb0370b4181006fb81b6 I0406 02:57:30.341632 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=218fc0467b953cd3c4480409e236a0d4fce80e1a86e3e07841192d85904125f9 I0406 02:57:30.341757 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=a718fbcfe35981bce7689823ce827de82e72b67c8c4407aee766fea5905d03b0 I0406 02:57:30.341801 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=f4b9e04f442efd64448805226187f8f2a7fcfe999900eb89e02f25699045f56d I0406 02:57:30.341836 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=79e39da76194a98b1ff8c9c9b7c6eceaf9f7d94eec867ce08389b41d8127d338 I0406 02:57:30.341872 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=b6b5b4abaeff077d46a7db7c6324e6487d9b0f39bc40e45737522c8aaf88ae0b I0406 02:57:30.341907 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=9c2714c9093faebbd9756883e993831aef17a1bf8b1144338cdf1a94fa5f0064 I0406 02:57:30.341944 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=9de27bc104c4e062998445dadc03c78d595bdf19620d402a9de49ea3e4935dab I0406 02:57:30.341973 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=fd81fb264b4ffd881dd6d3a9310bda5c971fd0c59e1b53774d0b25e8eb4e632d I0406 02:57:30.342020 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=e3d15f3dcfd8f66a19f5b2c6feb2ab212604bc33940b203ab7b64963704720a0 I0406 02:57:30.342057 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=15ac912e77f416acb1e5c23b68557d8c56b7e8c8220fc80ccabb1cf5409643eb I0406 02:57:30.342091 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=cd9a895e6cc6e71c41a315540a49584a1c02ada72138da5db82baf295b5a11aa I0406 02:57:30.342122 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=09d6d84b111f535bcffecce726f8218ac83cdc6dec5c4057deffd1b9b436ef14 I0406 02:57:30.342131 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 35.521877ms to process 13 records I0406 02:57:30.342217 1 tasks_processing.go:74] worker 15 stopped. I0406 02:57:30.342339 1 recorder.go:75] Recording config/authentication with fingerprint=9ec1da72a57d00a40a3961bd4261c6e33a2a05fed138e465b43653dfa8d63ef3 I0406 02:57:30.342353 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 33.174629ms to process 1 records I0406 02:57:30.342436 1 tasks_processing.go:74] worker 58 stopped. I0406 02:57:30.342566 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=47b49879fb257d4d0f4e0930ba70609436cf83f64df62155e33e838496961cd8 I0406 02:57:30.342576 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 32.817788ms to process 1 records I0406 02:57:30.342585 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 33.475247ms to process 0 records I0406 02:57:30.342589 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 37.32325ms to process 0 records I0406 02:57:30.342594 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 35.127006ms to process 0 records E0406 02:57:30.342599 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0406 02:57:30.342603 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 33.450353ms to process 0 records I0406 02:57:30.342651 1 tasks_processing.go:74] worker 6 stopped. I0406 02:57:30.342663 1 tasks_processing.go:74] worker 8 stopped. I0406 02:57:30.342669 1 tasks_processing.go:74] worker 13 stopped. I0406 02:57:30.342672 1 tasks_processing.go:74] worker 30 stopped. I0406 02:57:30.342742 1 tasks_processing.go:74] worker 63 stopped. I0406 02:57:30.342953 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=a5a5f58b291cd85b28596d56a9f84dd59b54fdde7a01c297637e4565e3c66cd9 I0406 02:57:30.342969 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 34.171567ms to process 1 records I0406 02:57:30.343680 1 tasks_processing.go:74] worker 9 stopped. I0406 02:57:30.343696 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 38.557187ms to process 0 records I0406 02:57:30.343704 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 38.609794ms to process 0 records I0406 02:57:30.343709 1 tasks_processing.go:74] worker 28 stopped. W0406 02:57:30.344069 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0406 02:57:30.344572 1 tasks_processing.go:74] worker 34 stopped. I0406 02:57:30.344706 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=12b20f9176599a660b77da76acfab3a3938c310c9c1d376e4a14b2ab9f6cdb80 I0406 02:57:30.344763 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=9ad163ede9feb9a0200863b80483dc93339b8a326d96e1d95050ba3fcaf1bd6f I0406 02:57:30.344788 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=35a956da917b843af9beebe5c328d67537989f8651cae6f7b50ceec168519c4a I0406 02:57:30.344807 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=95125e22a8793c3e8e628db5f73a8bee974e8d0b10cb9f33662e815d214ded3f I0406 02:57:30.344831 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=db8c6a67cd5a7e348b05e31a38347dd62ae7223b4c81664df5a2592e5cb291af I0406 02:57:30.344854 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=d67bec073da5d59de891b3d76aede42225fd27a07e6220f6a9aba3beb87a999b I0406 02:57:30.344929 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=185bbc1a150475518a905888931bae5a5e973a8346dd8baa900cf123024759b4 I0406 02:57:30.344952 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=1bbea1ea8181a7f685062cd61440c1172abe2f73b1fa6f71d1e1b87918982d0f I0406 02:57:30.344976 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=ff9b172a0ba92b50c5932a740fd78c36700b7e87f89bdb1d29ba4f15e1f554a4 I0406 02:57:30.345000 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=78407da4bbdf916cb2b9b1ab663f497305c63d9e3dca843bcf313d8734f46f35 I0406 02:57:30.345007 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 37.421126ms to process 10 records I0406 02:57:30.345613 1 tasks_processing.go:74] worker 27 stopped. I0406 02:57:30.346047 1 recorder.go:75] Recording config/node/ip-10-0-0-56.ec2.internal with fingerprint=c30fb488fb107457266a27b4f6c128d88952f839aa98b19fc25713673fd63844 I0406 02:57:30.346230 1 recorder.go:75] Recording config/node/ip-10-0-1-4.ec2.internal with fingerprint=1d4ee8ba874bc7c312943ab9400587c85599def115f36bddb81c198fbfca9fd4 I0406 02:57:30.346384 1 recorder.go:75] Recording config/node/ip-10-0-2-216.ec2.internal with fingerprint=46f939d1a409500cc70d392e9cb7554da05119acb5b7a721ada64187b2c823e5 I0406 02:57:30.346414 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 39.837116ms to process 3 records I0406 02:57:30.346628 1 tasks_processing.go:74] worker 48 stopped. I0406 02:57:30.346639 1 recorder.go:75] Recording config/network with fingerprint=07fdd701c9826cc978e8bdb03557ca6840bdcea8a6f2886f53fa3e63be5d340a I0406 02:57:30.346709 1 gather.go:177] gatherer "clusterconfig" function "networks" took 39.751586ms to process 1 records I0406 02:57:30.346813 1 tasks_processing.go:74] worker 51 stopped. I0406 02:57:30.347830 1 controller.go:254] Source scaController *sca.Controller is not ready I0406 02:57:30.347843 1 controller.go:254] Source clusterTransferController *clustertransfer.Controller is not ready I0406 02:57:30.347849 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0406 02:57:30.347852 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0406 02:57:30.347854 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0406 02:57:30.347871 1 controller.go:531] The operator is still being initialized I0406 02:57:30.347878 1 controller.go:554] The operator is healthy I0406 02:57:30.348251 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=b8d89e18534cfa1c1f39f8b8842e90e01326893af7e5634f32590bda3725998f I0406 02:57:30.348669 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=19d5363aeb7e896f5df1c6ba22d14e8d7eafa77b2bc3592bc9d0976cbc9eb652 I0406 02:57:30.348688 1 gather.go:177] gatherer "clusterconfig" function "crds" took 38.066208ms to process 2 records I0406 02:57:30.348698 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 41.004861ms to process 0 records I0406 02:57:30.348702 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 40.932748ms to process 0 records I0406 02:57:30.348709 1 tasks_processing.go:74] worker 44 stopped. I0406 02:57:30.348712 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0406 02:57:30.348716 1 tasks_processing.go:74] worker 14 stopped. I0406 02:57:30.348718 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 41.051025ms to process 1 records I0406 02:57:30.348721 1 tasks_processing.go:74] worker 1 stopped. I0406 02:57:30.348742 1 tasks_processing.go:74] worker 16 stopped. I0406 02:57:30.348781 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=07f3fba7acbadd3e8dc90882ef6a1cbc68948fd5148510a8c35b6e7148ec8756 I0406 02:57:30.348790 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 40.308975ms to process 1 records I0406 02:57:30.349089 1 tasks_processing.go:74] worker 62 stopped. E0406 02:57:30.349101 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0406 02:57:30.349117 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0406 02:57:30.349126 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 42.938475ms to process 1 records I0406 02:57:30.349131 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 42.151789ms to process 0 records I0406 02:57:30.349136 1 tasks_processing.go:74] worker 35 stopped. I0406 02:57:30.354903 1 tasks_processing.go:74] worker 20 stopped. I0406 02:57:30.355105 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=3f4a3a38e219acc281dfc5d24588ea9b875d3b9b77e32036231703d2a7c1d6a1 I0406 02:57:30.355184 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=9da01d98807a2962595bcd6e67ed68554f7b4828bda2b87248d06d47abe3003a I0406 02:57:30.355193 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 47.108936ms to process 2 records I0406 02:57:30.358974 1 tasks_processing.go:74] worker 25 stopped. I0406 02:57:30.359214 1 recorder.go:75] Recording config/version with fingerprint=40190093c9d5f9c92bb019e48b4e0cd5f29e9f4288f5b1139645bfca18e3aa97 I0406 02:57:30.359228 1 recorder.go:75] Recording config/id with fingerprint=e415f07702416a741369d3517439ebedf2a18b615a99377794b9f4ea00c6529a I0406 02:57:30.359234 1 gather.go:177] gatherer "clusterconfig" function "version" took 51.447341ms to process 2 records I0406 02:57:30.359338 1 tasks_processing.go:74] worker 50 stopped. I0406 02:57:30.359354 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 51.173179ms to process 0 records I0406 02:57:30.361690 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0406 02:57:30.372245 1 tasks_processing.go:74] worker 17 stopped. I0406 02:57:30.373439 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-57d9b95648-2mg48 with fingerprint=c20938b2b765e70e0480235e8b8400acce905863ba63981881059310c7c2e28e I0406 02:57:30.373514 1 recorder.go:75] Recording config/running_containers with fingerprint=0997db8dcd3f800f6d4fde053f5a8308b09eef237fd21f5db0ddeba8c33863ec I0406 02:57:30.373523 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 67.393465ms to process 2 records I0406 02:57:30.373534 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 64.554837ms to process 0 records E0406 02:57:30.373540 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0406 02:57:30.373544 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2pgcu9mfas1m46ulnh3binnrqpenib9u-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2pgcu9mfas1m46ulnh3binnrqpenib9u-primary-cert-bundle-secret" not found I0406 02:57:30.373576 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=f6b670a761e6e3f2afbb0fc3ccbe9b966e5dbad714b942f69b0ffa5adc7a76f0 I0406 02:57:30.373584 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 65.338636ms to process 1 records I0406 02:57:30.373591 1 tasks_processing.go:74] worker 54 stopped. I0406 02:57:30.373596 1 tasks_processing.go:74] worker 60 stopped. I0406 02:57:30.374281 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0406 02:57:30.374285 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0406 02:57:30.374293 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0406 02:57:30.377322 1 tasks_processing.go:74] worker 61 stopped. E0406 02:57:30.377335 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0406 02:57:30.377341 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0406 02:57:30.377345 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0406 02:57:30.377354 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=20ed6d7fdef919e1dbf14043d15a28454160f2e604706803e55a38d1623c3158 I0406 02:57:30.377385 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0406 02:57:30.377391 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0406 02:57:30.377406 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=0bcb269fd90b6e8af4a57174ee20ae48ebee01bb0a0c651000e372609b4ee122 I0406 02:57:30.377427 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0406 02:57:30.377465 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0406 02:57:30.377472 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0406 02:57:30.377477 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 68.355803ms to process 7 records I0406 02:57:30.378454 1 base_controller.go:82] Caches are synced for ConfigController I0406 02:57:30.378465 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0406 02:57:30.381768 1 configmapobserver.go:84] configmaps "insights-config" not found I0406 02:57:30.410676 1 base_controller.go:82] Caches are synced for LoggingSyncer I0406 02:57:30.410687 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0406 02:57:30.424907 1 cluster_transfer.go:102] no available accepted cluster transfer I0406 02:57:30.424919 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0406 02:57:30.506548 1 sca.go:266] etc-pki-entitlement secret successfully created I0406 02:57:30.506564 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated W0406 02:57:31.344112 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0406 02:57:32.343967 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0406 02:57:32.769944 1 tasks_processing.go:74] worker 7 stopped. I0406 02:57:32.770052 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=ffd9ec210c0c7316b75e163c62277ed16946d40327f03464b797fc3fc608d172 I0406 02:57:32.770096 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0406 02:57:32.770129 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=1b0da1031548bae77d01dc358adbd09e58ec9222beeb15001d4f9dfb98593c4f I0406 02:57:32.770139 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0406 02:57:32.770159 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=3a03ff9305b6a067016772eb5be86068dd789055898aec8e0411ed1edc588107 I0406 02:57:32.770175 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0406 02:57:32.770197 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=6965749e1ce1d3edaa0c3f5c84d85a3cf7f1bffa162ea861be7262a22aa8e1d6 I0406 02:57:32.770223 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=88082d7fedfab53a6ad5c819b81a1b0e48046a6c1ce2cd0fd73a27048c6a4e15 I0406 02:57:32.770246 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=edbbbc9e22302369bbe0a0b4aacf666896958108707abcb0c31252c6f066e3e1 I0406 02:57:32.770279 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=ec6649ddb3831c8e5a0b733dd6af746d61ac3ccea842933fffdbba6d2ec0b06f I0406 02:57:32.770561 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=e14ea9c5950a46711068e32985947db81c2671e31eecfd1cf6215e27fd93e854 I0406 02:57:32.770582 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=a1188e199be7c593651de6ce7df998a70a4706349c40bde3100e3047916e530d I0406 02:57:32.770592 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0406 02:57:32.770606 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=16c8f576eb16674f3df2d34cf79b858fa0dfa6bed0317f6095d605941db7b0ef I0406 02:57:32.770614 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0406 02:57:32.770629 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=95d5ca621d89bd9cf272fb5b0ad43e60f86f6f1643fd901c2064c6dac4bd545c I0406 02:57:32.770638 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0406 02:57:32.770656 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=0129a4f9a6499729c37edfb6c311623b1ad7efdb1ba44dc1dc96dab894ab9370 I0406 02:57:32.770666 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0406 02:57:32.770684 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=b17cab5c8761a8d97d3a8ee7e1c2d4cedb8535f17d75bbef93e8f15cfbb2a6f6 I0406 02:57:32.770796 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=f65c6ec994a0127720d1b7bf1ef74d97044e53590e488222da4f0a6a64a7fa7a I0406 02:57:32.770805 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0406 02:57:32.770812 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0406 02:57:32.770833 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0406 02:57:32.770854 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=1c4d213a899e0c48b2e63cbc5fe75b42a6c9e0f3b0ad28f07896be9b450fc259 I0406 02:57:32.770877 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=8dd5d726203fa205b6de4c2d79a6ad59bfc31d6551ac1f470ca19367bbed37e4 I0406 02:57:32.770886 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0406 02:57:32.770902 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=48aa7ec59ae061c5298f4e239fab02d49d1bfe7fcaed96a4cedefb3972587bfc I0406 02:57:32.770911 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0406 02:57:32.770928 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=fa36ccdcb3b5a6b1afcbd2e7e0a1aeb2f68b3b72ce9dcae530fd148793f5da3a I0406 02:57:32.770938 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0406 02:57:32.770949 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=b824ba13a2ae763c840de3298870273f0376426d573a5dc4bb48090b66febe31 I0406 02:57:32.770963 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=eebf65d17fc038c66cfcf27128b2053ce0464cd25f14c2b067d4250f3b53641d I0406 02:57:32.770976 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=b78140c2547fef939ee42827b49d7f1b5c4b9d0489be6e62621127b99a79d2f4 I0406 02:57:32.770996 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=9cc58c3098ee5bce78d7435ae4058587d645a2b00f37710529a981ecd81be58d I0406 02:57:32.771005 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0406 02:57:32.771031 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=7b963d8a2fb1e2e8f2b093b1e573354f800164d161133d59c1e808ef62cd7897 I0406 02:57:32.771058 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0406 02:57:32.771066 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0406 02:57:32.771074 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.462716604s to process 39 records W0406 02:57:33.344098 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0406 02:57:34.344188 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0406 02:57:35.344582 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0406 02:57:35.344606 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0406 02:57:35.344624 1 tasks_processing.go:74] worker 49 stopped. E0406 02:57:35.344637 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0406 02:57:35.344649 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0406 02:57:35.344658 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.036811218s to process 1 records I0406 02:57:43.152184 1 tasks_processing.go:74] worker 57 stopped. I0406 02:57:43.152229 1 recorder.go:75] Recording config/installplans with fingerprint=f17dbfacc3bfddf27ca3b213b39495434cd4c4e9e3dbd69566ffb3845bbcf539 I0406 02:57:43.152241 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.844574073s to process 1 records I0406 02:57:43.913716 1 tasks_processing.go:74] worker 59 stopped. I0406 02:57:43.913998 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=168fffbb2ffe3e4223178b9929987b11ee2f64fe6f0ebe7873bbab778d18dbe6 I0406 02:57:43.914015 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.608597017s to process 1 records E0406 02:57:43.914075 1 periodic.go:254] "Unhandled Error" err="clusterconfig failed after 13.609s with: function \"silenced_alerts\" failed with an error, function \"tsdb_status\" failed with an error, function \"active_alerts\" failed with an error, function \"metrics\" failed with an error, function \"support_secret\" failed with an error, function \"machines\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machine_configs\" failed with an error, function \"ingress_certificates\" failed with an error, function \"config_maps\" failed with an error, function \"dvo_metrics\" failed with an error" I0406 02:57:43.915172 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "silenced_alerts" failed with an error, function "tsdb_status" failed with an error, function "active_alerts" failed with an error, function "metrics" failed with an error, function "support_secret" failed with an error, function "machines" failed with an error, function "machine_healthchecks" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machine_configs" failed with an error, function "ingress_certificates" failed with an error, function "config_maps" failed with an error, function "dvo_metrics" failed with an error I0406 02:57:43.915187 1 periodic.go:216] Running workloads gatherer I0406 02:57:43.915201 1 tasks_processing.go:45] number of workers: 2 I0406 02:57:43.915206 1 tasks_processing.go:69] worker 1 listening for tasks. I0406 02:57:43.915210 1 tasks_processing.go:71] worker 1 working on workload_info task. I0406 02:57:43.915216 1 tasks_processing.go:69] worker 0 listening for tasks. I0406 02:57:43.915235 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0406 02:57:43.921238 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-56.ec2.internal... I0406 02:57:43.921244 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-216.ec2.internal... I0406 02:57:43.921255 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-4.ec2.internal... I0406 02:57:43.939528 1 tasks_processing.go:74] worker 0 stopped. I0406 02:57:43.939541 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 24.275052ms to process 0 records I0406 02:57:48.640636 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 5s I0406 02:57:48.664507 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0406 02:57:48.671395 1 gather_workloads_info.go:387] No image sha256:6552b42b5cec3ef82f4b7b30214f2b4e4561f83599f1a77341fe77b9a6171bc2 (9ms) I0406 02:57:48.679518 1 gather_workloads_info.go:387] No image sha256:8c9c050549f490f6b96eca68bb61a32f164b2152cae31b4ffb7d7ecc4bd678b1 (8ms) I0406 02:57:48.686918 1 gather_workloads_info.go:387] No image sha256:524c4410d51b126c6097fae3b2ad88ec67900e7b786a9131b7a8e96a194e4847 (7ms) I0406 02:57:48.694828 1 gather_workloads_info.go:387] No image sha256:014e2427899021034589f5602bb55916492eba8531030bd8a07bf1501c99f374 (8ms) I0406 02:57:48.702255 1 gather_workloads_info.go:387] No image sha256:1647489821e206e3ac1d3f1fbecdff3621ef87a7064f492906334c16f072a3e3 (7ms) I0406 02:57:48.710056 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (8ms) I0406 02:57:48.717410 1 gather_workloads_info.go:387] No image sha256:0b449cd4c58e81397bc99ba2d0970585d2a3cc1472a8134b46cf20e8c488b8c5 (7ms) I0406 02:57:48.724959 1 gather_workloads_info.go:387] No image sha256:89b38bc6c2b4405d467684858b8a4f8980f6ad9d721e7ca75e9956f923639240 (8ms) I0406 02:57:48.733081 1 gather_workloads_info.go:387] No image sha256:11b1019f1ef459a09826a731547c78028048bf66d9d35d3fadf20efa387c176c (8ms) I0406 02:57:48.740820 1 gather_workloads_info.go:387] No image sha256:5fd3dcbd54108018a438b849eb89f35b4c34343cb5319fbfb30cf252e2a58a8f (8ms) I0406 02:57:48.770791 1 gather_workloads_info.go:387] No image sha256:0c31c251fe9c6e08b68cf7ca3a04234a62d4da93043dafdb3a756cd78c70f11a (30ms) I0406 02:57:48.871072 1 gather_workloads_info.go:387] No image sha256:3b313209f88d2df827ff378cf0af36bd18908f1cab0e7ace14df1e8ffa5f1c00 (100ms) I0406 02:57:48.971919 1 gather_workloads_info.go:387] No image sha256:8a28cd812d21b11f8c7c145b9cc7ccd5de13efa5e57bc13003764d776aba69dd (101ms) I0406 02:57:49.072081 1 gather_workloads_info.go:387] No image sha256:1aa00d2bc23cc0d2699bc526bd537f9f4dd86d1f1460b04e927244da59f55a10 (100ms) I0406 02:57:49.171257 1 gather_workloads_info.go:387] No image sha256:1010e0ad0ad46c299a0074b6bfd09a1889a6671ab1c3979a78f679a22e1ee328 (99ms) I0406 02:57:49.272106 1 gather_workloads_info.go:387] No image sha256:0932593cd3bb6cd2e2b9ecad5e3bbe2fdcd107785ab3f8b25596f4901d8ae33a (101ms) I0406 02:57:49.371374 1 gather_workloads_info.go:387] No image sha256:709239c59cc22bd41c553a7f30c147c4fb33ed6f560cba9ff976a966cc58dc3c (99ms) I0406 02:57:49.471460 1 gather_workloads_info.go:387] No image sha256:46eddf2c3239c92218286368ced60b42cea6f789a09ab9981f0e6fbd216cb1f6 (100ms) I0406 02:57:49.572115 1 gather_workloads_info.go:387] No image sha256:3c2ae4e3bd8501436ec135700a13a6ed29ad6080cd151fd932646639787148a0 (101ms) I0406 02:57:49.671280 1 gather_workloads_info.go:387] No image sha256:07c82fce5e470c7ee9c3c1fdc6c953c962ed119411c88bf5fd4f0a08e3a0b9b7 (99ms) I0406 02:57:49.771614 1 gather_workloads_info.go:387] No image sha256:9ac7550a7edd59fdbc0e5b3d336defbd6fe0510087272b78d364a4580fd3d020 (100ms) I0406 02:57:49.873190 1 gather_workloads_info.go:387] No image sha256:62075f81a8bc4ce5a7192023a80f2e27c2bc4fae547193776513d67fbddad601 (102ms) I0406 02:57:49.972316 1 gather_workloads_info.go:387] No image sha256:5ae6198359a92af6c4b317806ec8a33ff53d44189d14561a8f8a3153d7afdb81 (99ms) I0406 02:57:50.072065 1 gather_workloads_info.go:387] No image sha256:326a8b5302b495cb059364ab96dfdd38425252613cfa3abb7e1dacaecda4fb1e (100ms) I0406 02:57:50.173358 1 gather_workloads_info.go:398] Found image sha256:2bcdce58a9dec8306ea7ec58ed588c6def1d0e2891d06b91661cb04a1411242f (101ms) I0406 02:57:50.272169 1 gather_workloads_info.go:387] No image sha256:5e61f9895dd3c359917209aaa56cb9a1fba5e7b7946348f44ff1301045f0d1f0 (99ms) I0406 02:57:50.372134 1 gather_workloads_info.go:387] No image sha256:17eab4abb2d95e2fe5dead3b0ccd144401f0f4467fc5f1dec8e19d5cb9bfd315 (100ms) I0406 02:57:50.471406 1 gather_workloads_info.go:387] No image sha256:9fed4b776332f3e8ce9c8454beabcb0c940b0e09c0c973eeafcf653b27a79043 (99ms) I0406 02:57:50.572115 1 gather_workloads_info.go:387] No image sha256:43e816c2f891134e527145559d700e6b393b9ccec1e3c51ab45dc3ba4ac5f266 (101ms) I0406 02:57:50.671383 1 gather_workloads_info.go:387] No image sha256:543891894646911f3714c3ebb7dd08f28cb0f0ea0d51045d058f7d2e9d475481 (99ms) I0406 02:57:50.772199 1 gather_workloads_info.go:387] No image sha256:4def742b6a5ef27c09ac9540e132622ec7178284d8900b9a844cb425e8be0c94 (101ms) I0406 02:57:50.872429 1 gather_workloads_info.go:387] No image sha256:5589d0d744b2f935e7f1bb3e094e813685092910a00e1853891b31fe9e330d23 (100ms) I0406 02:57:50.971968 1 gather_workloads_info.go:387] No image sha256:595b03c50302148c33de51825571a2b59a1930591d461a717315be05c6efdbb1 (100ms) I0406 02:57:51.072259 1 gather_workloads_info.go:387] No image sha256:b4fd7ff14b54766e058b847d7f91c13dc17bf661e45c07bca5bc9754d20e14ae (100ms) I0406 02:57:51.172256 1 gather_workloads_info.go:387] No image sha256:55cb1bfa110d67c2452259e6b4a2158b8f6e7c4bba9e7dfd661e0dea37ff2e25 (100ms) I0406 02:57:51.272989 1 gather_workloads_info.go:387] No image sha256:3d02603962035bdee1a13b1b0d17740cc9c5bcdc89f2372652e5a7dd907ca313 (101ms) I0406 02:57:51.372188 1 gather_workloads_info.go:387] No image sha256:7958abc201949f9faed6fe6cd4f8e2f8dab06dde35f09779cec19e9286f3983d (99ms) I0406 02:57:51.471928 1 gather_workloads_info.go:387] No image sha256:145ba54fe931a01cab8cef5d1d1535a1f3b719403b000e2b34256d7ede7ec6f8 (100ms) I0406 02:57:51.578372 1 gather_workloads_info.go:387] No image sha256:0bc5a5a619cb16e67f81551ea4b96af7a8db26addfb202d9ca5ceab4168f459f (106ms) I0406 02:57:51.672147 1 gather_workloads_info.go:387] No image sha256:2113fd46aa3b497a315f73e215a7c26f8f2930b5e5b64f74eee88a83db412df1 (94ms) I0406 02:57:51.771481 1 gather_workloads_info.go:387] No image sha256:2f4b0241dc0818547ffbaf42f8f17cac3ea8cde598a6c0bd3896872f53f7a3ff (99ms) I0406 02:57:51.872762 1 gather_workloads_info.go:387] No image sha256:4622e61163cba7f1fa87feddf2832e0c8cb4dae6fb70f41b6e03b39cb93770ff (101ms) I0406 02:57:51.972488 1 gather_workloads_info.go:387] No image sha256:1e0266333cbe4c6264f1783cea8781a4f6fc8691fbc1b2a95ed8979d7cc99ae7 (100ms) I0406 02:57:52.072354 1 gather_workloads_info.go:387] No image sha256:0644c26b6a8f4aa4435145653370dd3f8438c6e299523ff8570eb2858ac65d09 (100ms) I0406 02:57:52.171978 1 gather_workloads_info.go:387] No image sha256:2973ce1c3d9df0e94568fdc45b90f86d6d7c65fa3bf6c319219b9e213a684101 (100ms) I0406 02:57:52.272699 1 gather_workloads_info.go:398] Found image sha256:1c6f9f9f726be814274f19e66bd04d069236460a312faa274eebe757d33f889c (101ms) I0406 02:57:52.372622 1 gather_workloads_info.go:387] No image sha256:215264f061f8b0eeba17dc442cf5a6a3a754f67104bf271f61d2ccee30b7e663 (100ms) I0406 02:57:52.372655 1 tasks_processing.go:74] worker 1 stopped. I0406 02:57:52.372958 1 recorder.go:75] Recording config/workload_info with fingerprint=b84d4a72b1aa22e92486478e58ce3ba620ad3e277d265adca8e2c2e7106e695d I0406 02:57:52.372978 1 gather.go:177] gatherer "workloads" function "workload_info" took 8.457436794s to process 1 records I0406 02:57:52.372999 1 periodic.go:263] Periodic gather workloads completed in 8.457s I0406 02:57:52.373010 1 controllerstatus.go:80] name=periodic-workloads healthy=true reason= message= I0406 02:57:52.373017 1 periodic.go:216] Running conditional gatherer I0406 02:57:52.378080 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.21.7/gathering_rules I0406 02:57:52.525156 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer I0406 02:57:53.576228 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0406 02:57:53.581455 1 conditional_gatherer.go:392] cluster version is '4.21.7' I0406 02:57:53.581481 1 tasks_processing.go:45] number of workers: 3 I0406 02:57:53.581490 1 tasks_processing.go:69] worker 2 listening for tasks. I0406 02:57:53.581494 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0406 02:57:53.581502 1 tasks_processing.go:69] worker 0 listening for tasks. I0406 02:57:53.581513 1 tasks_processing.go:69] worker 1 listening for tasks. I0406 02:57:53.581517 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0406 02:57:53.581523 1 tasks_processing.go:71] worker 1 working on conditional_gatherer_rules task. I0406 02:57:53.581531 1 tasks_processing.go:74] worker 0 stopped. I0406 02:57:53.581599 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0406 02:57:53.581616 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.655µs to process 1 records I0406 02:57:53.581683 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=c8d4892cd4f2eea27d57353c6055eeb530bc2d811bfe041f6984d90395255a8c I0406 02:57:53.581694 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.372µs to process 1 records I0406 02:57:53.581750 1 tasks_processing.go:74] worker 2 stopped. I0406 02:57:53.581721 1 tasks_processing.go:74] worker 1 stopped. I0406 02:57:53.581761 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 219.94µs to process 0 records I0406 02:57:53.581782 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0406 02:57:53.581799 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0406 02:57:53.608477 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=f1740fe014a937af76419f222e6f85562407921ce068d171648320aaee35d097 I0406 02:57:53.608578 1 diskrecorder.go:70] Writing 107 records to /var/lib/insights-operator/insights-2026-04-06-025753.tar.gz I0406 02:57:53.615013 1 diskrecorder.go:51] Wrote 107 records to disk in 6ms I0406 02:57:53.615039 1 periodic.go:285] Gathering cluster info every 2h0m0s I0406 02:57:53.615057 1 periodic.go:286] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0406 02:59:00.305649 1 diskrecorder.go:223] Found files to send: insights-2026-04-06-025753.tar.gz I0406 02:59:00.305677 1 insightsuploader.go:150] Checking archives to upload periodically every 16m10.666739983s I0406 02:59:00.305685 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0406 02:59:00.318171 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0406 02:59:00.474393 1 requests.go:87] Successfully reported id=2026-04-06T02:59:00Z x-rh-insights-request-id=90b9aabb934b4343aac30d57da4844c3, wrote=47639 I0406 02:59:00.474443 1 insightsuploader.go:187] Uploaded report successfully in 168.749864ms I0406 02:59:00.474462 1 controller.go:129] Initializing last reported time to 2026-04-06T02:59:00Z I0406 02:59:00.474502 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0406 02:59:00.474511 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0406 02:59:00.474516 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0406 02:59:00.480317 1 controller.go:554] The operator is healthy I0406 02:59:30.316336 1 controller.go:554] The operator is healthy I0406 03:00:02.091969 1 insightsreport.go:137] Pulling report from smart-proxy I0406 03:00:02.091992 1 insightsreport.go:149] Retrieving report I0406 03:00:02.098748 1 requests.go:110] Retrieving report for cluster: a961b014-d9ab-4487-a97e-3270d9af00f5 I0406 03:00:02.098760 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/a961b014-d9ab-4487-a97e-3270d9af00f5/reports I0406 03:00:02.101704 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/a961b014-d9ab-4487-a97e-3270d9af00f5/reports I0406 03:00:02.559981 1 insightsreport.go:184] Report retrieved I0406 03:00:02.587207 1 insightsreport.go:239] Report retrieved correctly I0406 03:01:30.317763 1 controller.go:554] The operator is healthy I0406 03:02:30.300575 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0406 03:02:30.305714 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0406 03:02:30.305743 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0406 03:02:30.309489 1 secretconfigobserver.go:119] support secret does not exist I0406 03:03:30.320862 1 controller.go:554] The operator is healthy I0406 03:05:30.319360 1 controller.go:554] The operator is healthy I0406 03:07:30.310520 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0406 03:07:30.459267 1 configmapobserver.go:84] configmaps "insights-config" not found I0406 03:07:30.509252 1 controller.go:554] The operator is healthy I0406 03:07:30.509648 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0406 03:07:30.509669 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0406 03:07:30.596177 1 configmapobserver.go:84] configmaps "insights-config" not found I0406 03:07:30.610852 1 secretconfigobserver.go:119] support secret does not exist I0406 03:09:30.319646 1 controller.go:554] The operator is healthy I0406 03:11:30.319377 1 controller.go:554] The operator is healthy I0406 03:12:30.610959 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0406 03:12:30.619869 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0406 03:12:30.619886 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0406 03:12:30.623502 1 secretconfigobserver.go:119] support secret does not exist I0406 03:13:30.320414 1 controller.go:554] The operator is healthy I0406 03:15:16.142151 1 insightsuploader.go:146] Nothing to report since 2026-04-06T02:59:00Z I0406 03:15:30.318717 1 controller.go:554] The operator is healthy I0406 03:17:30.294949 1 configmapobserver.go:84] configmaps "insights-config" not found I0406 03:17:30.299022 1 configmapobserver.go:84] configmaps "insights-config" not found I0406 03:17:30.316173 1 controller.go:554] The operator is healthy I0406 03:17:30.624487 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0406 03:17:30.628792 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0406 03:17:30.628808 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0406 03:17:30.632425 1 secretconfigobserver.go:119] support secret does not exist I0406 03:19:30.316235 1 controller.go:554] The operator is healthy I0406 03:21:30.316878 1 controller.go:554] The operator is healthy