I0420 11:40:36.515200 1 cmd.go:253] Using service-serving-cert provided certificates I0420 11:40:36.515652 1 observer_polling.go:159] Starting file observer I0420 11:40:36.902397 1 operator.go:76] Starting insights-operator v0.0.0-master+$Format:%H$ I0420 11:40:36.902630 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0420 11:40:36.903140 1 secure_serving.go:57] Forcing use of http/1.1 only W0420 11:40:36.903163 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0420 11:40:36.903169 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0420 11:40:36.903176 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0420 11:40:36.903181 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0420 11:40:36.903186 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0420 11:40:36.903190 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0420 11:40:36.903209 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0420 11:40:36.908316 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"5a2ae0c8-7ab1-4a6d-b958-b905d86ac550", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "HyperShiftOnlyDynamicResourceAllocation", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryption", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLM", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "OpenShiftPodSecurityAdmission", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}} I0420 11:40:36.907820 1 operator.go:141] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNSInstall GatewayAPI GatewayAPIController HighlyAvailableArbiter HyperShiftOnlyDynamicResourceAllocation ImageStreamImportMode ImageVolume KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS ManagedBootImagesAzure ManagedBootImagesvSphere MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages PreconfiguredUDNAddresses ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SigstoreImageVerification SigstoreImageVerificationPKI StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSDualStackInstall AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureDualStackInstall AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement CBORServingAndStorage CRDCompatibilityRequirementOperator ClientsAllowCBOR ClientsPreferCBOR ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterAPIMachineManagement ClusterAPIMachineManagementVSphere ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud EtcdBackendQuota EventTTL EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall GCPDualStackInstall ImageModeStatusReporting IngressControllerDynamicConfigurationManager InsightsConfig InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryption KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesCPMS MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutableCSINodeAllocatableCount MutatingAdmissionPolicy NewOLM NewOLMBoxCutterRuntime NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterInstall NutanixMultiSubnets OSStreams OVNObservability OnPremDNSRecords OpenShiftPodSecurityAdmission ProvisioningRequestAvailable SELinuxMount ShortCertRotation SignatureStores TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeGroupSnapshot] I0420 11:40:36.908772 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0420 11:40:36.908791 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0420 11:40:36.908781 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0420 11:40:36.908806 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0420 11:40:36.908802 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0420 11:40:36.908827 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0420 11:40:36.909040 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0420 11:40:36.909306 1 secure_serving.go:213] Serving securely on [::]:8443 I0420 11:40:36.909334 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0420 11:40:36.912726 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0420 11:40:36.912758 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0420 11:40:36.912844 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0420 11:40:36.917154 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 11:40:36.917171 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0420 11:40:36.921176 1 secretconfigobserver.go:119] support secret does not exist I0420 11:40:36.925132 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 11:40:36.928594 1 secretconfigobserver.go:119] support secret does not exist I0420 11:40:36.930684 1 recorder.go:176] Pruning old reports every 4h25m48s, max age is 288h0m0s I0420 11:40:36.934612 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0420 11:40:36.934620 1 periodic.go:216] Running clusterconfig gatherer I0420 11:40:36.934628 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0420 11:40:36.934644 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0420 11:40:36.934649 1 insightsreport.go:296] Starting report retriever I0420 11:40:36.934653 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0420 11:40:36.934675 1 tasks_processing.go:45] number of workers: 64 I0420 11:40:36.934696 1 tasks_processing.go:69] worker 0 listening for tasks. I0420 11:40:36.934700 1 tasks_processing.go:69] worker 1 listening for tasks. I0420 11:40:36.934709 1 tasks_processing.go:69] worker 11 listening for tasks. I0420 11:40:36.934710 1 tasks_processing.go:69] worker 3 listening for tasks. I0420 11:40:36.934711 1 tasks_processing.go:69] worker 5 listening for tasks. I0420 11:40:36.934715 1 tasks_processing.go:69] worker 6 listening for tasks. I0420 11:40:36.934720 1 tasks_processing.go:69] worker 2 listening for tasks. I0420 11:40:36.934719 1 tasks_processing.go:69] worker 7 listening for tasks. I0420 11:40:36.934724 1 tasks_processing.go:69] worker 14 listening for tasks. I0420 11:40:36.934729 1 tasks_processing.go:69] worker 4 listening for tasks. I0420 11:40:36.934733 1 tasks_processing.go:69] worker 12 listening for tasks. I0420 11:40:36.934725 1 tasks_processing.go:69] worker 8 listening for tasks. I0420 11:40:36.934739 1 tasks_processing.go:69] worker 10 listening for tasks. I0420 11:40:36.934732 1 tasks_processing.go:69] worker 9 listening for tasks. I0420 11:40:36.934744 1 tasks_processing.go:69] worker 13 listening for tasks. I0420 11:40:36.934749 1 tasks_processing.go:69] worker 15 listening for tasks. I0420 11:40:36.934751 1 tasks_processing.go:69] worker 20 listening for tasks. I0420 11:40:36.934751 1 tasks_processing.go:69] worker 16 listening for tasks. I0420 11:40:36.934757 1 tasks_processing.go:69] worker 17 listening for tasks. I0420 11:40:36.934742 1 tasks_processing.go:69] worker 18 listening for tasks. I0420 11:40:36.934761 1 tasks_processing.go:69] worker 31 listening for tasks. I0420 11:40:36.934762 1 tasks_processing.go:69] worker 19 listening for tasks. I0420 11:40:36.934764 1 tasks_processing.go:69] worker 22 listening for tasks. I0420 11:40:36.934770 1 tasks_processing.go:69] worker 28 listening for tasks. I0420 11:40:36.934772 1 tasks_processing.go:69] worker 48 listening for tasks. I0420 11:40:36.934772 1 tasks_processing.go:69] worker 32 listening for tasks. I0420 11:40:36.934772 1 tasks_processing.go:69] worker 54 listening for tasks. I0420 11:40:36.934778 1 tasks_processing.go:69] worker 25 listening for tasks. I0420 11:40:36.934778 1 tasks_processing.go:69] worker 27 listening for tasks. I0420 11:40:36.934786 1 tasks_processing.go:69] worker 49 listening for tasks. I0420 11:40:36.934790 1 tasks_processing.go:71] worker 27 working on node_features task. I0420 11:40:36.934791 1 tasks_processing.go:69] worker 37 listening for tasks. I0420 11:40:36.934793 1 tasks_processing.go:69] worker 53 listening for tasks. I0420 11:40:36.934797 1 tasks_processing.go:69] worker 50 listening for tasks. I0420 11:40:36.934800 1 tasks_processing.go:71] worker 14 working on active_alerts task. I0420 11:40:36.934802 1 tasks_processing.go:71] worker 17 working on machine_autoscalers task. I0420 11:40:36.934806 1 tasks_processing.go:69] worker 45 listening for tasks. I0420 11:40:36.934809 1 tasks_processing.go:69] worker 34 listening for tasks. I0420 11:40:36.934806 1 tasks_processing.go:69] worker 39 listening for tasks. I0420 11:40:36.934802 1 tasks_processing.go:69] worker 33 listening for tasks. I0420 11:40:36.934815 1 tasks_processing.go:71] worker 10 working on clusterroles task. I0420 11:40:36.934818 1 tasks_processing.go:69] worker 35 listening for tasks. I0420 11:40:36.935385 1 tasks_processing.go:71] worker 6 working on node_logs task. I0420 11:40:36.934779 1 tasks_processing.go:69] worker 52 listening for tasks. I0420 11:40:36.934791 1 tasks_processing.go:69] worker 24 listening for tasks. I0420 11:40:36.934750 1 tasks_processing.go:69] worker 23 listening for tasks. I0420 11:40:36.934779 1 tasks_processing.go:69] worker 51 listening for tasks. I0420 11:40:36.934780 1 tasks_processing.go:69] worker 29 listening for tasks. I0420 11:40:36.935427 1 tasks_processing.go:71] worker 50 working on nodenetworkstates task. I0420 11:40:36.935461 1 tasks_processing.go:71] worker 2 working on support_secret task. I0420 11:40:36.935509 1 tasks_processing.go:71] worker 7 working on sap_pods task. I0420 11:40:36.934807 1 tasks_processing.go:69] worker 46 listening for tasks. I0420 11:40:36.935578 1 tasks_processing.go:69] worker 26 listening for tasks. I0420 11:40:36.935591 1 tasks_processing.go:71] worker 13 working on ceph_cluster task. I0420 11:40:36.934788 1 tasks_processing.go:71] worker 25 working on install_plans task. I0420 11:40:36.935610 1 tasks_processing.go:71] worker 34 working on jaegers task. I0420 11:40:36.934783 1 tasks_processing.go:71] worker 54 working on certificate_signing_requests task. I0420 11:40:36.935669 1 tasks_processing.go:71] worker 9 working on validating_webhook_configurations task. I0420 11:40:36.934765 1 tasks_processing.go:69] worker 21 listening for tasks. I0420 11:40:36.935688 1 tasks_processing.go:71] worker 23 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0420 11:40:36.934782 1 tasks_processing.go:71] worker 32 working on qemu_kubevirt_launcher_logs task. I0420 11:40:36.935694 1 tasks_processing.go:71] worker 35 working on cost_management_metrics_configs task. I0420 11:40:36.934808 1 tasks_processing.go:69] worker 40 listening for tasks. I0420 11:40:36.935753 1 tasks_processing.go:71] worker 40 working on version task. I0420 11:40:36.935865 1 tasks_processing.go:71] worker 26 working on mutating_webhook_configurations task. I0420 11:40:36.936050 1 tasks_processing.go:71] worker 51 working on machine_config_pools task. I0420 11:40:36.934782 1 tasks_processing.go:69] worker 42 listening for tasks. I0420 11:40:36.934787 1 tasks_processing.go:69] worker 30 listening for tasks. I0420 11:40:36.934791 1 tasks_processing.go:69] worker 55 listening for tasks. I0420 11:40:36.934796 1 tasks_processing.go:69] worker 38 listening for tasks. I0420 11:40:36.934796 1 tasks_processing.go:69] worker 44 listening for tasks. I0420 11:40:36.934765 1 tasks_processing.go:69] worker 41 listening for tasks. I0420 11:40:36.934797 1 tasks_processing.go:71] worker 4 working on crds task. I0420 11:40:36.935011 1 tasks_processing.go:69] worker 59 listening for tasks. I0420 11:40:36.934789 1 tasks_processing.go:69] worker 43 listening for tasks. I0420 11:40:36.934807 1 tasks_processing.go:71] worker 12 working on infrastructures task. I0420 11:40:36.936335 1 tasks_processing.go:71] worker 39 working on olm_operators task. I0420 11:40:36.936693 1 tasks_processing.go:71] worker 46 working on operators task. I0420 11:40:36.936736 1 tasks_processing.go:71] worker 59 working on sap_config task. I0420 11:40:36.935690 1 tasks_processing.go:71] worker 21 working on image_registries task. I0420 11:40:36.935986 1 tasks_processing.go:71] worker 33 working on pdbs task. I0420 11:40:36.934811 1 tasks_processing.go:71] worker 8 working on operators_pods_and_events task. I0420 11:40:36.934816 1 tasks_processing.go:69] worker 47 listening for tasks. I0420 11:40:36.934819 1 tasks_processing.go:69] worker 56 listening for tasks. I0420 11:40:36.935303 1 tasks_processing.go:69] worker 61 listening for tasks. I0420 11:40:36.936854 1 tasks_processing.go:71] worker 61 working on image_pruners task. I0420 11:40:36.935069 1 tasks_processing.go:71] worker 18 working on machine_sets task. I0420 11:40:36.934812 1 tasks_processing.go:69] worker 63 listening for tasks. I0420 11:40:36.935310 1 tasks_processing.go:71] worker 3 working on storage_classes task. I0420 11:40:36.934817 1 tasks_processing.go:71] worker 19 working on nodenetworkconfigurationpolicies task. I0420 11:40:36.935070 1 tasks_processing.go:71] worker 0 working on proxies task. I0420 11:40:36.937165 1 tasks_processing.go:71] worker 52 working on machine_healthchecks task. I0420 11:40:36.935292 1 tasks_processing.go:71] worker 37 working on config_maps task. I0420 11:40:36.934802 1 tasks_processing.go:71] worker 16 working on metrics task. I0420 11:40:36.935125 1 tasks_processing.go:71] worker 31 working on monitoring_persistent_volumes task. I0420 11:40:36.935317 1 tasks_processing.go:69] worker 62 listening for tasks. I0420 11:40:36.935351 1 tasks_processing.go:69] worker 60 listening for tasks. I0420 11:40:36.936002 1 tasks_processing.go:71] worker 29 working on tsdb_status task. I0420 11:40:36.936468 1 tasks_processing.go:69] worker 58 listening for tasks. I0420 11:40:36.936477 1 tasks_processing.go:71] worker 24 working on openstack_controlplanes task. I0420 11:40:36.935123 1 tasks_processing.go:71] worker 15 working on sap_datahubs task. I0420 11:40:36.935061 1 tasks_processing.go:69] worker 36 listening for tasks. I0420 11:40:36.937287 1 tasks_processing.go:71] worker 43 working on openstack_dataplanenodesets task. I0420 11:40:36.937966 1 tasks_processing.go:71] worker 36 working on overlapping_namespace_uids task. I0420 11:40:36.937988 1 tasks_processing.go:71] worker 63 working on ingress_certificates task. I0420 11:40:36.938071 1 tasks_processing.go:71] worker 30 working on schedulers task. I0420 11:40:36.938127 1 tasks_processing.go:71] worker 38 working on ingress task. I0420 11:40:36.935318 1 tasks_processing.go:71] worker 5 working on silenced_alerts task. I0420 11:40:36.935325 1 tasks_processing.go:71] worker 45 working on storage_cluster task. I0420 11:40:36.935129 1 tasks_processing.go:71] worker 1 working on container_images task. I0420 11:40:36.938190 1 tasks_processing.go:71] worker 44 working on openstack_dataplanedeployments task. I0420 11:40:36.938256 1 tasks_processing.go:71] worker 41 working on nodes task. I0420 11:40:36.938074 1 tasks_processing.go:71] worker 58 working on machines task. I0420 11:40:36.935190 1 tasks_processing.go:71] worker 22 working on container_runtime_configs task. I0420 11:40:36.935187 1 tasks_processing.go:71] worker 28 working on openshift_machine_api_events task. I0420 11:40:36.935194 1 tasks_processing.go:71] worker 48 working on aggregated_monitoring_cr_names task. I0420 11:40:36.935200 1 tasks_processing.go:69] worker 57 listening for tasks. I0420 11:40:36.938563 1 tasks_processing.go:71] worker 57 working on networks task. I0420 11:40:36.935197 1 tasks_processing.go:71] worker 20 working on feature_gates task. I0420 11:40:36.935247 1 tasks_processing.go:71] worker 53 working on dvo_metrics task. I0420 11:40:36.938659 1 tasks_processing.go:71] worker 50 working on cluster_apiserver task. I0420 11:40:36.938691 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 3.193876ms to process 0 records I0420 11:40:36.935250 1 tasks_processing.go:71] worker 49 working on pod_network_connectivity_checks task. I0420 11:40:36.937540 1 tasks_processing.go:71] worker 62 working on machine_configs task. I0420 11:40:36.938085 1 tasks_processing.go:71] worker 60 working on authentication task. I0420 11:40:36.938798 1 tasks_processing.go:71] worker 47 working on openshift_logging task. I0420 11:40:36.935205 1 tasks_processing.go:71] worker 11 working on openstack_version task. I0420 11:40:36.938869 1 tasks_processing.go:71] worker 56 working on image task. I0420 11:40:36.938962 1 tasks_processing.go:71] worker 55 working on oauths task. I0420 11:40:36.939272 1 tasks_processing.go:71] worker 42 working on lokistack task. I0420 11:40:36.939906 1 controller.go:129] Initializing last reported time to 0001-01-01T00:00:00Z I0420 11:40:36.939920 1 controller.go:368] The initial operator extension status is healthy I0420 11:40:36.939928 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0420 11:40:36.939932 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0420 11:40:36.939935 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0420 11:40:36.939949 1 controller.go:531] The operator is still being initialized I0420 11:40:36.939954 1 controller.go:554] The operator is healthy I0420 11:40:36.940022 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0420 11:40:36.940075 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0420 11:40:36.940161 1 operator.go:328] started I0420 11:40:36.940188 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer E0420 11:40:36.942738 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.176.193:9091: connect: connection refused I0420 11:40:36.942757 1 tasks_processing.go:71] worker 29 working on service_accounts task. E0420 11:40:36.942748 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.176.193:9091: connect: connection refused E0420 11:40:36.942773 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.176.193:9091: connect: connection refused I0420 11:40:36.942797 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 4.662942ms to process 0 records E0420 11:40:36.942815 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.176.193:9091: connect: connection refused I0420 11:40:36.942831 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 5.542358ms to process 0 records I0420 11:40:36.942846 1 tasks_processing.go:74] worker 16 stopped. I0420 11:40:36.943674 1 tasks_processing.go:74] worker 35 stopped. I0420 11:40:36.943686 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 7.968227ms to process 0 records I0420 11:40:36.951028 1 tasks_processing.go:74] worker 18 stopped. I0420 11:40:36.951041 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 14.106608ms to process 0 records I0420 11:40:36.951049 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 14.937297ms to process 0 records I0420 11:40:36.951053 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 15.406223ms to process 0 records I0420 11:40:36.951059 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 15.51561ms to process 0 records I0420 11:40:36.951062 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 16.244152ms to process 0 records I0420 11:40:36.951066 1 tasks_processing.go:74] worker 17 stopped. I0420 11:40:36.951070 1 tasks_processing.go:74] worker 7 stopped. I0420 11:40:36.951074 1 tasks_processing.go:74] worker 51 stopped. I0420 11:40:36.951073 1 tasks_processing.go:74] worker 34 stopped. I0420 11:40:36.951072 1 tasks_processing.go:74] worker 15 stopped. I0420 11:40:36.951080 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 13.491355ms to process 0 records I0420 11:40:36.951091 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 13.097289ms to process 0 records I0420 11:40:36.951095 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 12.949183ms to process 0 records I0420 11:40:36.951097 1 tasks_processing.go:74] worker 43 stopped. I0420 11:40:36.951100 1 tasks_processing.go:74] worker 24 stopped. I0420 11:40:36.951153 1 tasks_processing.go:74] worker 45 stopped. I0420 11:40:36.951166 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 12.869626ms to process 0 records I0420 11:40:36.951180 1 tasks_processing.go:74] worker 59 stopped. I0420 11:40:36.951188 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 14.429087ms to process 0 records I0420 11:40:36.951196 1 tasks_processing.go:74] worker 58 stopped. E0420 11:40:36.951204 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0420 11:40:36.951211 1 gather.go:177] gatherer "clusterconfig" function "machines" took 12.915885ms to process 0 records I0420 11:40:36.951537 1 tasks_processing.go:74] worker 61 stopped. I0420 11:40:36.951793 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=467ed76d405983b63aad0695c67c9c25de7ec82f9d50cc0582cbd1491e3fca1a I0420 11:40:36.951811 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 14.669424ms to process 1 records I0420 11:40:36.958054 1 tasks_processing.go:74] worker 0 stopped. I0420 11:40:36.958117 1 recorder.go:75] Recording config/proxy with fingerprint=1ad55ea458d21d56ec71b2267c427b8b4b43c8285b6a944f48ad6526988701d0 I0420 11:40:36.958127 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 20.913833ms to process 1 records I0420 11:40:36.958216 1 tasks_processing.go:74] worker 38 stopped. I0420 11:40:36.958339 1 recorder.go:75] Recording config/ingress with fingerprint=d49d365e4b4cc0f5d911273c99a3adc699ccdc0515d173f418e97f1adbcf2336 I0420 11:40:36.958350 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 19.965713ms to process 1 records I0420 11:40:36.958375 1 tasks_processing.go:74] worker 57 stopped. I0420 11:40:36.958500 1 recorder.go:75] Recording config/network with fingerprint=08debd1a08d08d63b98bf00fdd572ead44a21be0f6cd60d9cf51765a506a963e I0420 11:40:36.958514 1 gather.go:177] gatherer "clusterconfig" function "networks" took 19.7714ms to process 1 records E0420 11:40:36.962619 1 gather_node_features.go:86] GatherNodeFeatures: NodeFeatures resource not found in openshift-nfd namespace (may not be installed) I0420 11:40:36.962627 1 gather_logs.go:145] no pods in namespace were found I0420 11:40:36.962629 1 tasks_processing.go:74] worker 27 stopped. I0420 11:40:36.962633 1 gather.go:177] gatherer "clusterconfig" function "node_features" took 27.833218ms to process 0 records I0420 11:40:36.962640 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 26.934781ms to process 0 records I0420 11:40:36.962644 1 tasks_processing.go:74] worker 32 stopped. I0420 11:40:36.962651 1 tasks_processing.go:74] worker 47 stopped. I0420 11:40:36.962658 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 23.8308ms to process 0 records I0420 11:40:36.967895 1 tasks_processing.go:74] worker 44 stopped. I0420 11:40:36.967908 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 29.6957ms to process 0 records I0420 11:40:36.968323 1 tasks_processing.go:74] worker 11 stopped. I0420 11:40:36.968341 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 29.458336ms to process 0 records I0420 11:40:36.968350 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 29.884699ms to process 0 records I0420 11:40:36.968363 1 tasks_processing.go:74] worker 22 stopped. I0420 11:40:36.969163 1 tasks_processing.go:74] worker 60 stopped. I0420 11:40:36.969944 1 recorder.go:75] Recording config/authentication with fingerprint=a6f86ede52f7926798e4083aeece0477e827fac5b4d22fb155ad4a496de69faf I0420 11:40:36.969964 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 30.444326ms to process 1 records I0420 11:40:36.970550 1 tasks_processing.go:74] worker 12 stopped. I0420 11:40:36.971042 1 recorder.go:75] Recording config/infrastructure with fingerprint=72d079c0ecc547cc44fe4dadd165a90bed809ee6f8a5ce37d4dbc02b3f6195a8 I0420 11:40:36.971055 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 33.937208ms to process 1 records I0420 11:40:36.971140 1 recorder.go:75] Recording config/featuregate with fingerprint=e70388139e1f9a8bd63adb5615f8c0c42ba65dfc1a8e4cf810058d5e44b85219 I0420 11:40:36.971148 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 31.968916ms to process 1 records I0420 11:40:36.971151 1 tasks_processing.go:74] worker 20 stopped. I0420 11:40:36.971188 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=7278fd222adfffef612e871c6e3fee4272c5401933b4832478bf27600328e97b I0420 11:40:36.971197 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 32.507978ms to process 1 records I0420 11:40:36.971225 1 tasks_processing.go:74] worker 30 stopped. I0420 11:40:36.971285 1 tasks_processing.go:74] worker 50 stopped. I0420 11:40:36.971299 1 recorder.go:75] Recording config/apiserver with fingerprint=3de54f7f9f2d4a4ab22d63c85ca0cbb2b1701314b2316bd70383fb9f9cfac89e I0420 11:40:36.971307 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 32.513239ms to process 1 records I0420 11:40:36.974899 1 tasks_processing.go:74] worker 28 stopped. I0420 11:40:36.974917 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 36.426528ms to process 0 records I0420 11:40:36.975219 1 tasks_processing.go:74] worker 31 stopped. I0420 11:40:36.975240 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 37.807179ms to process 0 records I0420 11:40:36.975761 1 tasks_processing.go:74] worker 19 stopped. I0420 11:40:36.975778 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 38.634489ms to process 0 records I0420 11:40:36.975807 1 tasks_processing.go:74] worker 42 stopped. I0420 11:40:36.975818 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 36.517181ms to process 0 records I0420 11:40:36.975853 1 tasks_processing.go:74] worker 13 stopped. I0420 11:40:36.975869 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 40.252946ms to process 0 records I0420 11:40:36.975964 1 tasks_processing.go:74] worker 49 stopped. E0420 11:40:36.975978 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0420 11:40:36.975990 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 37.247568ms to process 0 records I0420 11:40:36.976061 1 tasks_processing.go:74] worker 33 stopped. I0420 11:40:36.976159 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=b8d20371097d08365f4f54ca9728746d2ab7029e3ff5d9d2f1950f605584e36a I0420 11:40:36.976178 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=4a089cb3575d13d438804a0454c6f9ef3af559f58901b7e60ef812ea8c20ac03 I0420 11:40:36.976193 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=349f70c2a1d6b35e5645c5fefa130196c83115ba0cf6861d40137994a61d0dcb I0420 11:40:36.976211 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=8c00dc39df3e17aa793f74fde240ca377b59b5f66caaeb0c25d156a0c693f13e I0420 11:40:36.976227 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=3bf47fa47959373cbd3944d9f6f3851e9085e4498e9b85580148117de2edfb72 I0420 11:40:36.976239 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=18d653e0508cb20bd66f54175b7fabeaedb1f1458264c4de6bb9888821ad79fd I0420 11:40:36.976256 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=ca5a4094ed1d1772853a6188570c8dd348cdfa54afc3e51ebf3417c19c4df9f9 I0420 11:40:36.976271 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=3cf824710268c1ecaee55052eaf0d47b9a0bb998a14c4344fb7c65459cfeae7d I0420 11:40:36.976291 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=f41fe641d08b44f9c040e8db557fb0f8b6e99acd1522f873776143e678b9de01 I0420 11:40:36.976308 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=ec83c8f563a3ea7d2d0221cf78d7a0d5ececb795aeea6cde6e9e39664709a6b7 I0420 11:40:36.976655 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=ff959ad4d4f1a8f4b77fead12baddedb1026abc5984bd4852a596d2e275a66d0 I0420 11:40:36.976677 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 39.230748ms to process 11 records I0420 11:40:36.976740 1 tasks_processing.go:74] worker 56 stopped. I0420 11:40:36.977101 1 recorder.go:75] Recording config/image with fingerprint=6500214f12f7864aaa0e861d27aed9ed9301340a4c5a4b76e364d674fe49bb48 I0420 11:40:36.977150 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0420 11:40:36.977182 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0420 11:40:36.977188 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0420 11:40:36.977194 1 controller.go:254] Source scaController *sca.Controller is not ready I0420 11:40:36.977198 1 controller.go:254] Source clusterTransferController *clustertransfer.Controller is not ready I0420 11:40:36.977304 1 controller.go:531] The operator is still being initialized I0420 11:40:36.977359 1 controller.go:554] The operator is healthy I0420 11:40:36.977166 1 gather.go:177] gatherer "clusterconfig" function "image" took 37.335605ms to process 1 records E0420 11:40:36.977461 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0420 11:40:36.977474 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 40.77229ms to process 0 records I0420 11:40:36.977643 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=a40d8960979bc0619d68fa99d300a4f1f1183cec6b9ed04d8077919b84ee0631 I0420 11:40:36.977701 1 tasks_processing.go:74] worker 2 stopped. I0420 11:40:36.977731 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=e6fe7c9695b1d50ae7ee9ee39902efdadade1a8d0847639e7d0fa2398a6b27cf I0420 11:40:36.977745 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 39.836692ms to process 2 records I0420 11:40:36.977753 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 41.711305ms to process 0 records E0420 11:40:36.977764 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0420 11:40:36.977916 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 40.318294ms to process 0 records I0420 11:40:36.977933 1 tasks_processing.go:74] worker 52 stopped. I0420 11:40:36.977939 1 tasks_processing.go:74] worker 3 stopped. I0420 11:40:36.977943 1 tasks_processing.go:74] worker 6 stopped. I0420 11:40:36.978286 1 tasks_processing.go:74] worker 55 stopped. I0420 11:40:36.979093 1 recorder.go:75] Recording config/oauth with fingerprint=34a6747dc04d292262c05dab6b980051416b427ce868c5b48ab65268cb63b240 I0420 11:40:36.979118 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 39.011132ms to process 1 records W0420 11:40:36.979683 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0420 11:40:36.980333 1 tasks_processing.go:74] worker 21 stopped. I0420 11:40:36.981349 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=c05ad1bbc58b9fa9efdf4ca2c3261f616a96407b9c89528cab82e89978733d47 I0420 11:40:36.981811 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 43.540761ms to process 1 records I0420 11:40:36.981912 1 tasks_processing.go:74] worker 41 stopped. I0420 11:40:36.982338 1 recorder.go:75] Recording config/node/ip-10-0-0-6.ec2.internal with fingerprint=a79d074694ee444a5f6cc1ec0cc6b744c6dc46c69297f80d17f121b93208a8c1 I0420 11:40:36.982477 1 recorder.go:75] Recording config/node/ip-10-0-1-82.ec2.internal with fingerprint=020eb622c35c64bda49723d1f40ef5798e59db290fe53610e1550ecbca28ff5f I0420 11:40:36.982575 1 recorder.go:75] Recording config/node/ip-10-0-2-103.ec2.internal with fingerprint=e3a287d43f24b901c439df455e6017e4eed4af32da6636dc43f6930b598a9aa3 I0420 11:40:36.982588 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 42.098434ms to process 3 records I0420 11:40:36.982723 1 tasks_processing.go:74] worker 40 stopped. I0420 11:40:36.983002 1 recorder.go:75] Recording config/version with fingerprint=33fb694880679f0056ee2f011f0446b1d90410c7ce5a91d2e05bcd26406a89d9 I0420 11:40:36.983032 1 recorder.go:75] Recording config/id with fingerprint=47f86eebd269c0adfa692add84c034fb19737bd31e554e9723b8da50c1d2f80b I0420 11:40:36.983044 1 gather.go:177] gatherer "clusterconfig" function "version" took 44.963131ms to process 2 records E0420 11:40:36.983056 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0420 11:40:36.983096 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0420 11:40:36.983111 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 42.063081ms to process 1 records I0420 11:40:36.983128 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0420 11:40:36.983136 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 43.008265ms to process 1 records I0420 11:40:36.983145 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 45.900452ms to process 0 records I0420 11:40:36.983149 1 tasks_processing.go:74] worker 62 stopped. I0420 11:40:36.983155 1 tasks_processing.go:74] worker 8 stopped. I0420 11:40:36.983161 1 tasks_processing.go:74] worker 36 stopped. I0420 11:40:36.983871 1 tasks_processing.go:74] worker 39 stopped. I0420 11:40:36.983949 1 recorder.go:75] Recording config/olm_operators with fingerprint=d4f889f9ce5cf64d372e36e0156cd15280a8241654ed1d2c1de386c61cfe25cd I0420 11:40:36.983977 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 47.171035ms to process 1 records I0420 11:40:36.985006 1 tasks_processing.go:74] worker 54 stopped. I0420 11:40:36.985027 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 49.359086ms to process 0 records I0420 11:40:36.987177 1 tasks_processing.go:74] worker 48 stopped. I0420 11:40:36.987196 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 48.677361ms to process 0 records I0420 11:40:36.991639 1 tasks_processing.go:74] worker 10 stopped. I0420 11:40:36.991893 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=aaa3b4db7137a2749ebd206585de480910f0a03bb2417ca0f539ed9afbb8a30e I0420 11:40:36.991982 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=be9ab48851d356d7894ba49095f5f574d20c8811bacee9b5312daea19e3f8882 I0420 11:40:36.991990 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 56.812281ms to process 2 records I0420 11:40:36.992030 1 tasks_processing.go:74] worker 4 stopped. I0420 11:40:36.992572 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=2904c8267d86798b0272df88e0cd4576cfe0a2839c3a70c276645d667d6fe5c8 I0420 11:40:36.992766 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=61c123b129de62f694436bb82f1b235c43a669f2bc6a2afcaa655e772eed2399 I0420 11:40:36.992776 1 gather.go:177] gatherer "clusterconfig" function "crds" took 55.81579ms to process 2 records I0420 11:40:36.993169 1 tasks_processing.go:74] worker 26 stopped. I0420 11:40:36.993349 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=c1fb51ed8ed171ede6ca4fad8eab1780c8b87f295c9b307e2b533ee2d0f180d8 I0420 11:40:36.993379 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=5aff921b85e913821945afaafee9c4d97432893f4f6855297a079d15c6290f73 I0420 11:40:36.993402 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=c612968181295b2f8dcd982348c82e88d6c018391ae7c2557214b5e7c6b21127 I0420 11:40:36.993411 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 57.126446ms to process 3 records I0420 11:40:36.998736 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0420 11:40:37.001519 1 tasks_processing.go:74] worker 1 stopped. I0420 11:40:37.002921 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-66dbbb6468-w5xz9 with fingerprint=0143bb97cdcb8c1e003c3f3c424d38fdcfb39d50fb11c9174f61ed9690bb59e8 I0420 11:40:37.003021 1 recorder.go:75] Recording config/running_containers with fingerprint=332ef745510a4fc84c3ee34562e60b7eee7001224d73cf6962e59f40d4fa1f75 I0420 11:40:37.003033 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 63.318334ms to process 2 records I0420 11:40:37.003106 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=ec906123ce0f2ffc487452d857c2512579e26ca81cf3b6ee836911d75352b4ec I0420 11:40:37.003115 1 tasks_processing.go:74] worker 9 stopped. I0420 11:40:37.003416 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=8cdfea7d8c63148f4c31da61fff764df1ac04c7763c66fc65c96f393ba3ec01d I0420 11:40:37.003607 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=f33d84e49cf121bbf807f2832e76f4bb70108908baf75a6f1b2099024c9f20fd I0420 11:40:37.003662 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=77a3fbeca5556e3faf58795c4f5c338727b30bbdd844d7e332619a53d405a74b I0420 11:40:37.003723 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=c21e46aab992544101843799c863ce16209a9e72d67bc43bfdaa7f9029525ac5 I0420 11:40:37.003774 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=95eb39a33cc6f1c8b3d71ef81592a7e71dcbadc3a3dc2125de22164bec7c0b42 I0420 11:40:37.003833 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=9095ec409d4c5c5b210d46972968bca94137654895f62509dbefe453df8a6f3f I0420 11:40:37.003925 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=19f0d56fabf1ed8083bb4cf6d23d3213aaf980ed46bf17a8206feef7740b76d4 I0420 11:40:37.003980 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=3335eb19dd77f8d442e5964ab370e968497b7e97ed5cdf6ba00d588360e19de4 I0420 11:40:37.004074 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=404c16ffd4655027369a36ad9edbd0156bac4ad163c05904c8c1e5aca248c560 I0420 11:40:37.004127 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=fdefee230f0392f08986770776dafe38b6c2622aa0999a645eeea634adf1dd2a I0420 11:40:37.004170 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=af38a84f32485223d0768d6f3996de7e9e13b1bf5afbe212d1dd9491ddeb26da I0420 11:40:37.004222 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=a6b5a35b7314c4b144e61aa9da3491aca538ba862c3f71887684e4854b9db2e5 I0420 11:40:37.004240 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 66.268429ms to process 13 records E0420 11:40:37.004256 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0420 11:40:37.004268 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2pprtcr8ssi9i0be4v4c4mp3tfmtq8os-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2pprtcr8ssi9i0be4v4c4mp3tfmtq8os-primary-cert-bundle-secret" not found I0420 11:40:37.004362 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=7e716058f6220061529de7e0b97a6b86d45983351baf11322913bc2bf50023d3 I0420 11:40:37.004380 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 65.637859ms to process 1 records I0420 11:40:37.004399 1 tasks_processing.go:74] worker 63 stopped. I0420 11:40:37.008832 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0420 11:40:37.008848 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0420 11:40:37.008880 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0420 11:40:37.009330 1 tasks_processing.go:74] worker 23 stopped. I0420 11:40:37.009345 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 73.626234ms to process 0 records I0420 11:40:37.010828 1 tasks_processing.go:74] worker 37 stopped. E0420 11:40:37.010842 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0420 11:40:37.010851 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0420 11:40:37.010855 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0420 11:40:37.010864 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=ad286723f58bdcfc37aeba3ec5b4110c08e1af59cd34d14b4bfaab02d18e4856 I0420 11:40:37.010915 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0420 11:40:37.010923 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0420 11:40:37.010927 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=2ab5ab7b1b10d7fcf1197bb24dea7c90f400e4effc18c7356873209d54fdf84b I0420 11:40:37.010938 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0420 11:40:37.010978 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0420 11:40:37.010985 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0420 11:40:37.010991 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 73.638326ms to process 7 records I0420 11:40:37.013606 1 base_controller.go:82] Caches are synced for ConfigController I0420 11:40:37.013616 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0420 11:40:37.017538 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 11:40:37.040425 1 base_controller.go:82] Caches are synced for LoggingSyncer I0420 11:40:37.040451 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0420 11:40:37.041537 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0420 11:40:37.171167 1 sca.go:266] etc-pki-entitlement secret successfully created I0420 11:40:37.171190 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated I0420 11:40:37.345861 1 cluster_transfer.go:102] no available accepted cluster transfer I0420 11:40:37.345880 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer W0420 11:40:37.977133 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0420 11:40:38.020405 1 tasks_processing.go:74] worker 5 stopped. I0420 11:40:38.020423 1 recorder.go:75] Recording config/silenced_alerts.json with fingerprint=37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570 I0420 11:40:38.020452 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 1.082193474s to process 1 records I0420 11:40:38.020744 1 tasks_processing.go:74] worker 14 stopped. I0420 11:40:38.020785 1 recorder.go:75] Recording config/alerts with fingerprint=7f9db3943f54bcc2f000e53ad5bf0f8e4bfcd77b50ff123402eae984479c997b I0420 11:40:38.020812 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 1.085928293s to process 1 records W0420 11:40:38.977101 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0420 11:40:39.407957 1 tasks_processing.go:74] worker 46 stopped. I0420 11:40:39.408069 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=fbce253cbaf37d089f56fed58acae6e3428992c86769fa92da6e32bc1849ba84 I0420 11:40:39.408113 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0420 11:40:39.408143 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=3fce87a995b7b54e3cc0f78acb027f78e0d3e49e312b180fd6ded39e246e2bd9 I0420 11:40:39.408153 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0420 11:40:39.408175 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=3942aa41ec2eab09f7f4e891a6f0fd2979b3427ece45f32418fd1195e789a353 I0420 11:40:39.408190 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0420 11:40:39.408213 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=bbd550fe45aa0dcd9d9967590041a36d4f1391d41d033b7aa35daaa87ba1682e I0420 11:40:39.408241 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=852f0860f2ad705bf892d421e7fd44170e3e317e234598bd9d32538a40710646 I0420 11:40:39.408266 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=508be0e9ea510268a0d10a4cd139bfc3e1ae2dcae40b1a1c7a40f269a521332c I0420 11:40:39.408295 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=08cd6abe6fa2b3ad4d24db77af695036414b255cec336acb4c399738b88d0761 I0420 11:40:39.408583 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=cf9c44e69c6185b5c738743f813c9a8434a4072e30f0f427205235c02ca0f3a1 I0420 11:40:39.408604 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=4b72dcb4d82604892944f21dc9395e1fae11eecabb798f1a0a185b79655b423d I0420 11:40:39.408611 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0420 11:40:39.408627 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=c035614f1ba8d4445473a0cf06a8e18cdf93b23586e5d665b6e8e66fb2d678a2 I0420 11:40:39.408638 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0420 11:40:39.408652 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=36b8a31a4be63ef94d18983a1814cd673ed349ee7d51d49bb79480bd4396bd8b I0420 11:40:39.408660 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0420 11:40:39.408678 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=731fc786a6672580b968deaea95fe5620419dba7cc9bd04572efea3ac4debec5 I0420 11:40:39.408687 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0420 11:40:39.408706 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=b5c970e75d39bbbff12f3cdc02ec314e957686eb8b1e8553301a1b356b3dd7b7 I0420 11:40:39.408824 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=6ee53471a88f5d89a80f6b576a7baa91d485518e6d9f9f1163bbf6c15990358f I0420 11:40:39.408834 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0420 11:40:39.408840 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0420 11:40:39.408862 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0420 11:40:39.408884 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=6c0f42b9ca45ec6df03ba6e0278b42352bf5eb7c7ba02a4e7ccde006a81e3051 I0420 11:40:39.408908 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=c826ba7a2eba6b98e053181084f4c7cbf84917c5c521e9cb1d5050844c46ac01 I0420 11:40:39.408917 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0420 11:40:39.408932 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=2e61a2337badb18880978060546e78614c176dec36688e9f5f9ff75b8ebe2baf I0420 11:40:39.408942 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0420 11:40:39.408960 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=0ebe8b90c5700178e1152fedcfd3a745550086ab363b2860c70bd55351109b65 I0420 11:40:39.408969 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0420 11:40:39.408982 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=757a5bd8c34ab9fcb71ff3172aa62de062cddbd2ea289e023370942a615ff1b4 I0420 11:40:39.408996 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=fb48da0bfce076b7a0962cdf8b7eb5b772ff7a7bf5be9deb8669a400c3ebf7b0 I0420 11:40:39.409010 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=c19e2ecfc78793d9606524b9ad64f6145ec3cd088d85c175d28378c49a99ded4 I0420 11:40:39.409032 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=6fe695108e18e344e82255c370e165dda9f16d44e93df2fa09cd1b813d0f6f3a I0420 11:40:39.409042 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0420 11:40:39.409064 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=267d5f21abd0eb6777ec388dc45673aec6cfa29f0e27ebba3952bfe9e19270d8 I0420 11:40:39.409080 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0420 11:40:39.409086 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0420 11:40:39.409093 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.471242085s to process 39 records W0420 11:40:39.977877 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0420 11:40:40.978189 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0420 11:40:41.975373 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0420 11:40:41.975460 1 tasks_processing.go:74] worker 53 stopped. E0420 11:40:41.975479 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0420 11:40:41.975500 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0420 11:40:41.975516 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.036798463s to process 1 records I0420 11:40:49.787330 1 tasks_processing.go:74] worker 25 stopped. I0420 11:40:49.787386 1 recorder.go:75] Recording config/installplans with fingerprint=f17dbfacc3bfddf27ca3b213b39495434cd4c4e9e3dbd69566ffb3845bbcf539 I0420 11:40:49.787402 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.851717929s to process 1 records I0420 11:40:50.549023 1 tasks_processing.go:74] worker 29 stopped. I0420 11:40:50.549313 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=168fffbb2ffe3e4223178b9929987b11ee2f64fe6f0ebe7873bbab778d18dbe6 I0420 11:40:50.549335 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.606250825s to process 1 records E0420 11:40:50.549414 1 periodic.go:254] "Unhandled Error" err="clusterconfig failed after 13.614s with: function \"tsdb_status\" failed with an error, function \"metrics\" failed with an error, function \"machines\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"support_secret\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"machine_configs\" failed with an error, function \"ingress_certificates\" failed with an error, function \"config_maps\" failed with an error, function \"dvo_metrics\" failed with an error" I0420 11:40:50.551085 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "tsdb_status" failed with an error, function "metrics" failed with an error, function "machines" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "support_secret" failed with an error, function "machine_healthchecks" failed with an error, function "machine_configs" failed with an error, function "ingress_certificates" failed with an error, function "config_maps" failed with an error, function "dvo_metrics" failed with an error I0420 11:40:50.551101 1 periodic.go:216] Running workloads gatherer I0420 11:40:50.551117 1 tasks_processing.go:45] number of workers: 2 I0420 11:40:50.551125 1 tasks_processing.go:69] worker 1 listening for tasks. I0420 11:40:50.551130 1 tasks_processing.go:71] worker 1 working on workload_info task. I0420 11:40:50.551139 1 tasks_processing.go:69] worker 0 listening for tasks. I0420 11:40:50.551162 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0420 11:40:50.557047 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-82.ec2.internal... I0420 11:40:50.557057 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-103.ec2.internal... I0420 11:40:50.557045 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-6.ec2.internal... I0420 11:40:50.579946 1 tasks_processing.go:74] worker 0 stopped. I0420 11:40:50.579964 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 28.765472ms to process 0 records I0420 11:40:54.529892 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 4s I0420 11:40:54.555129 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0420 11:40:54.563472 1 gather_workloads_info.go:387] No image sha256:730d1b6988025bef0daa3a9a5d8467ec4a26b0382cc52f91c3375b4590d3518a (10ms) I0420 11:40:54.572383 1 gather_workloads_info.go:387] No image sha256:a043239802b3eb8b323d285193d2527fad0ecec98ca91d188a3472a2fac8ae04 (9ms) I0420 11:40:54.581054 1 gather_workloads_info.go:387] No image sha256:000105ef5150e7079b90a613fb9e6193e2a6ef9b1908d2dce44f2395d4fd070f (9ms) I0420 11:40:54.589958 1 gather_workloads_info.go:387] No image sha256:52db05084f2a2e7d7bf4040e4865df88ec4074efbbe59a78d861be2c5ab2aa40 (9ms) I0420 11:40:54.598820 1 gather_workloads_info.go:387] No image sha256:0b2e6536a0ca3afc5e64a218087a6287c74b77a359075a385f1949ca5bd51b22 (9ms) I0420 11:40:54.607287 1 gather_workloads_info.go:387] No image sha256:0a4dfb8d4c1b3849319d45b4c54dff26a7238a2c08fcaa121f93073e95ab12e8 (8ms) I0420 11:40:54.615680 1 gather_workloads_info.go:387] No image sha256:572ebf7c7fab1a518582f3ca46c866730423cce6e8a19f614dc15fd61bf0c65a (8ms) I0420 11:40:54.624406 1 gather_workloads_info.go:387] No image sha256:934d8e8c50f3c609b8eea80d1051111fe3d066fe8c65c79572072ae55fcb0a86 (9ms) I0420 11:40:54.633474 1 gather_workloads_info.go:387] No image sha256:55b1db6038c5beaed54c626e3343b7a8589cc0be8dc41d1a66b4deab766ff520 (9ms) I0420 11:40:54.642541 1 gather_workloads_info.go:387] No image sha256:5d6a2b8d2508c7da71192661e7c0dcf1335f48b6336a20189abffbe79eeff962 (9ms) I0420 11:40:54.663118 1 gather_workloads_info.go:387] No image sha256:25c148fd380b1a9db3f6039d2e0eabc489a954921452391390ba9192b2325678 (21ms) I0420 11:40:54.764047 1 gather_workloads_info.go:387] No image sha256:8f881f6f0498eaba860d711dc8e9dc066db8728f22abedb82e1dc9a8e1c30046 (101ms) I0420 11:40:54.864302 1 gather_workloads_info.go:387] No image sha256:9ef6dc311b880a6442b7a3a19006ddd955c7da36850bf000f5c9b68fd0862dd4 (100ms) I0420 11:40:54.964722 1 gather_workloads_info.go:398] Found image sha256:23204877f166455ce8f9de9b91a467eeda576f172d5f96124a68aa986c4775d6 (100ms) I0420 11:40:55.063965 1 gather_workloads_info.go:387] No image sha256:5940cc34b322325618929b1e0eb3864dae73d54c8f5b474e218dfb28123a61c7 (99ms) I0420 11:40:55.163984 1 gather_workloads_info.go:387] No image sha256:0acfe677ed58031f8381cb8a4f2d8917ec4cfa95485e2a46529dffae54c75ace (100ms) I0420 11:40:55.263982 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (100ms) I0420 11:40:55.364988 1 gather_workloads_info.go:387] No image sha256:48e883a3932aea9457f55cd4628d21397d429d4828f42f95d1c903d9d5395bde (101ms) I0420 11:40:55.463579 1 gather_workloads_info.go:387] No image sha256:89277d8d4560d71db88c2dcc67c992a24544ca21810920b609c1d49d53b4a287 (99ms) I0420 11:40:55.563924 1 gather_workloads_info.go:387] No image sha256:15e1b5f763553e86d0230a751b0783e3b990438fd07cf96eed164bccb72ed564 (100ms) I0420 11:40:55.663538 1 gather_workloads_info.go:387] No image sha256:ab60623bb32f7e75fca71ef65137731cae347a21c7a4091dfd583fa00732721c (100ms) I0420 11:40:55.763648 1 gather_workloads_info.go:387] No image sha256:17aefdc2f4752c23dc8f0947bb01fc34635822eeb18f2d8a5eddf8211b84d450 (100ms) I0420 11:40:55.863505 1 gather_workloads_info.go:387] No image sha256:5aaea0419169e55832cc27acfe0fe3b9513a343d6bdf71d3da1575ed322245d0 (100ms) I0420 11:40:55.965688 1 gather_workloads_info.go:398] Found image sha256:673ebc8cc22c56c8f410e011b2fa950d28cf7b6420e17fdb6580d6cb10523384 (102ms) I0420 11:40:56.063654 1 gather_workloads_info.go:387] No image sha256:5b6212b8f539f08e78417d8a4b7485ca0b4e7927cacd7b752742a28841bc8ccd (98ms) I0420 11:40:56.163102 1 gather_workloads_info.go:387] No image sha256:5ff204630794311b0b37fa7b197b933ad85d76a481bf7cdb3bcbada08f0cdcbf (99ms) I0420 11:40:56.264161 1 gather_workloads_info.go:387] No image sha256:5c6d21c3f97366bc7ab57031cc027b67405a684bd804ce364ed5998b0685eaca (101ms) I0420 11:40:56.364510 1 gather_workloads_info.go:387] No image sha256:15928a5b803fa00781b408dbc4c3bfa7c5287b18584ba31f817def8a4430c725 (100ms) I0420 11:40:56.464086 1 gather_workloads_info.go:387] No image sha256:5ac9c549d65fc1d8bc900773bebee43e9192bcec1bb5fa46afb4597230c16ac7 (100ms) I0420 11:40:56.563550 1 gather_workloads_info.go:387] No image sha256:bdeb1c455cffccdb6c90db238e9d371de06340b1d6105d47f66d87293f878ac1 (99ms) I0420 11:40:56.663834 1 gather_workloads_info.go:387] No image sha256:6c6d412ccf0c3d6a14645a79df6c4658c0f8725d7954cb83c3d2f0116499345a (100ms) I0420 11:40:56.764338 1 gather_workloads_info.go:387] No image sha256:53ff8efdd8f563421ebc1859ab021a3bbc1306f82d4ff5d9c21bc60ec5f57898 (100ms) I0420 11:40:56.862820 1 gather_workloads_info.go:387] No image sha256:084aa9b0f8a6d478549dc384d4e66da13ee9b25cc98531da861cc19dee2a9e8f (98ms) I0420 11:40:56.963538 1 gather_workloads_info.go:387] No image sha256:3bba1358d4a0ae878ff491c0c2cbfffe60649e110b40342b878fe8fa332f8858 (101ms) I0420 11:40:57.063584 1 gather_workloads_info.go:387] No image sha256:875c77e5d144f03fb91d8cee0259f6966683ca88d1bf818dbf4652c16b70312c (100ms) I0420 11:40:57.163173 1 gather_workloads_info.go:387] No image sha256:1eb6a6ee3c66ac15be6bc21bb6015c595c11c15b9dfd21bd9f5ce7f7b990ebaa (100ms) I0420 11:40:57.263645 1 gather_workloads_info.go:387] No image sha256:d64bea34bf3e1bb0b3a701c3ff14e66665afc1b050f28124ad7e6888eaec3a81 (100ms) I0420 11:40:57.363510 1 gather_workloads_info.go:387] No image sha256:08c5a78c8a5af04c549e2273aaf4bb452a75bf038d68aa9d01bb2aff66c30e90 (100ms) I0420 11:40:57.463984 1 gather_workloads_info.go:387] No image sha256:637d41f067a5239096fc22b135181cda5113da833f1370e7a73965e83792e93a (100ms) I0420 11:40:57.564592 1 gather_workloads_info.go:387] No image sha256:5808401268394502d335281ea1a294b07210461b986b58f91d7d1f29c0029c6d (101ms) I0420 11:40:57.663587 1 gather_workloads_info.go:387] No image sha256:6814f5fced0ef219d06374011c68a11a3da788a764a00a69ade435466d9ee240 (99ms) I0420 11:40:57.763997 1 gather_workloads_info.go:387] No image sha256:48f5559439bea030a6423652da448c25fff0c62ba293eb521843c1a8eb89ddb0 (100ms) I0420 11:40:57.864528 1 gather_workloads_info.go:387] No image sha256:9d6d63dce784df5863e6a0ba4c11317a26c9d756d4e62d47759c97190a71cce5 (101ms) I0420 11:40:57.965070 1 gather_workloads_info.go:387] No image sha256:15677f0b70e6aa2dfaf088e45fc1a425c22bf6fda326b8116f87e88b6694dfab (101ms) I0420 11:40:58.065508 1 gather_workloads_info.go:387] No image sha256:3380fecebaeee139c1607fbcd9346cebc92806679568e2244a8905ef0f52aad2 (100ms) I0420 11:40:58.164151 1 gather_workloads_info.go:387] No image sha256:30e597ec5d6bb96ff70a4f8688c748b659cd4fd5d73d222e8701821d236795c5 (99ms) I0420 11:40:58.164183 1 tasks_processing.go:74] worker 1 stopped. I0420 11:40:58.164549 1 recorder.go:75] Recording config/workload_info with fingerprint=98091c6e67803d43d973ff77c8be6abf15c768a8cf8db86472fb192a39296c48 I0420 11:40:58.164566 1 gather.go:177] gatherer "workloads" function "workload_info" took 7.613046297s to process 1 records I0420 11:40:58.164579 1 periodic.go:263] Periodic gather workloads completed in 7.613s I0420 11:40:58.164589 1 controllerstatus.go:80] name=periodic-workloads healthy=true reason= message= I0420 11:40:58.164594 1 periodic.go:216] Running conditional gatherer I0420 11:40:58.169416 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.21.9/gathering_rules I0420 11:40:58.428794 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer E0420 11:40:58.431321 1 conditional_gatherer.go:324] unable to update alerts cache: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?match%5B%5D=ALERTS%7Balertstate%3D%22firing%22%7D&query=ALERTS": dial tcp 172.30.176.193:9091: connect: connection refused I0420 11:40:58.431394 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0420 11:40:58.442199 1 conditional_gatherer.go:392] cluster version is '4.21.9' E0420 11:40:58.442214 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0420 11:40:58.442220 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0420 11:40:58.442223 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0420 11:40:58.442227 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0420 11:40:58.442229 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0420 11:40:58.442232 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0420 11:40:58.442235 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0420 11:40:58.442238 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0420 11:40:58.442240 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0420 11:40:58.442246 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0420 11:40:58.442259 1 tasks_processing.go:45] number of workers: 3 I0420 11:40:58.442274 1 tasks_processing.go:69] worker 2 listening for tasks. I0420 11:40:58.442276 1 tasks_processing.go:69] worker 0 listening for tasks. I0420 11:40:58.442283 1 tasks_processing.go:69] worker 1 listening for tasks. I0420 11:40:58.442288 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0420 11:40:58.442280 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0420 11:40:58.442314 1 tasks_processing.go:74] worker 0 stopped. I0420 11:40:58.442325 1 tasks_processing.go:71] worker 1 working on conditional_gatherer_rules task. I0420 11:40:58.442345 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0420 11:40:58.442362 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.248µs to process 1 records I0420 11:40:58.442431 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=9fafdcbf373f1f367f1d69551ea59114592cce1681189be0f3dde774e0530661 I0420 11:40:58.442454 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.423µs to process 1 records I0420 11:40:58.442455 1 tasks_processing.go:74] worker 1 stopped. I0420 11:40:58.442553 1 tasks_processing.go:74] worker 2 stopped. I0420 11:40:58.442566 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 251.261µs to process 0 records I0420 11:40:58.442591 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0420 11:40:58.442609 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0420 11:40:58.476004 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=cd5b1ccd3fb120131c42b9d14d3c7b024ca2ce41c53ac9a729d59cc57c6781a2 I0420 11:40:58.476163 1 diskrecorder.go:70] Writing 111 records to /var/lib/insights-operator/insights-2026-04-20-114058.tar.gz I0420 11:40:58.483096 1 diskrecorder.go:51] Wrote 111 records to disk in 7ms I0420 11:40:58.483125 1 periodic.go:285] Gathering cluster info every 2h0m0s I0420 11:40:58.483138 1 periodic.go:286] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0420 11:42:06.935002 1 diskrecorder.go:223] Found files to send: insights-2026-04-20-114058.tar.gz I0420 11:42:06.935038 1 insightsuploader.go:150] Checking archives to upload periodically every 15m52.39577938s I0420 11:42:06.935049 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0420 11:42:06.944150 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0420 11:42:07.087027 1 requests.go:87] Successfully reported id=2026-04-20T11:42:06Z x-rh-insights-request-id=35ea30362ccf492b97d4448f59685c7f, wrote=48313 I0420 11:42:07.087085 1 insightsuploader.go:187] Uploaded report successfully in 152.02972ms I0420 11:42:07.087107 1 controller.go:129] Initializing last reported time to 2026-04-20T11:42:06Z I0420 11:42:07.087139 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0420 11:42:07.087148 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0420 11:42:07.087155 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0420 11:42:07.092276 1 controller.go:554] The operator is healthy I0420 11:42:36.948544 1 controller.go:554] The operator is healthy I0420 11:43:07.535226 1 insightsreport.go:137] Pulling report from smart-proxy I0420 11:43:07.535252 1 insightsreport.go:149] Retrieving report I0420 11:43:07.541856 1 requests.go:110] Retrieving report for cluster: f2dcbde3-4719-423f-9cb4-46d02df8f33f I0420 11:43:07.541867 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/f2dcbde3-4719-423f-9cb4-46d02df8f33f/reports I0420 11:43:07.544887 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/f2dcbde3-4719-423f-9cb4-46d02df8f33f/reports I0420 11:43:07.876815 1 insightsreport.go:184] Report retrieved I0420 11:43:07.904184 1 insightsreport.go:239] Report retrieved correctly I0420 11:44:36.949628 1 controller.go:554] The operator is healthy I0420 11:45:36.928938 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 11:45:36.937220 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 11:45:36.937238 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 11:45:36.941649 1 secretconfigobserver.go:119] support secret does not exist I0420 11:46:36.999007 1 controller.go:554] The operator is healthy I0420 11:48:36.946680 1 controller.go:554] The operator is healthy I0420 11:50:36.931309 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 11:50:36.935315 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 11:50:36.942528 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 11:50:36.945600 1 controller.go:554] The operator is healthy I0420 11:50:36.946662 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 11:50:36.946679 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 11:50:36.950698 1 secretconfigobserver.go:119] support secret does not exist I0420 11:52:36.949284 1 controller.go:554] The operator is healthy I0420 11:54:36.950139 1 controller.go:554] The operator is healthy I0420 11:55:36.950792 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 11:55:36.959250 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 11:55:36.959267 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 11:55:36.963403 1 secretconfigobserver.go:119] support secret does not exist I0420 11:56:36.946075 1 controller.go:554] The operator is healthy I0420 11:58:04.483611 1 insightsuploader.go:146] Nothing to report since 2026-04-20T11:42:06Z I0420 11:58:36.947137 1 controller.go:554] The operator is healthy I0420 12:00:36.930200 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 12:00:36.934448 1 configmapobserver.go:84] configmaps "insights-config" not found I0420 12:00:36.945899 1 controller.go:554] The operator is healthy I0420 12:00:36.963747 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0420 12:00:36.968612 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0420 12:00:36.968628 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0420 12:00:36.972782 1 secretconfigobserver.go:119] support secret does not exist I0420 12:02:36.947116 1 controller.go:554] The operator is healthy I0420 12:04:36.946805 1 controller.go:554] The operator is healthy