I0504 20:45:11.405406 1 cmd.go:253] Using service-serving-cert provided certificates I0504 20:45:11.405861 1 observer_polling.go:159] Starting file observer I0504 20:45:11.844280 1 operator.go:76] Starting insights-operator v0.0.0-master+$Format:%H$ I0504 20:45:11.844556 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0504 20:45:11.845479 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0504 20:45:11.845515 1 secure_serving.go:57] Forcing use of http/1.1 only W0504 20:45:11.845536 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0504 20:45:11.845541 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0504 20:45:11.845547 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0504 20:45:11.845551 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0504 20:45:11.845555 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0504 20:45:11.845558 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0504 20:45:11.849280 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"87d818e1-fbe4-49ce-809e-dc35ed0fad5e", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "HyperShiftOnlyDynamicResourceAllocation", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryption", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLM", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "OpenShiftPodSecurityAdmission", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}} I0504 20:45:11.849283 1 operator.go:141] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNSInstall GatewayAPI GatewayAPIController HighlyAvailableArbiter HyperShiftOnlyDynamicResourceAllocation ImageStreamImportMode ImageVolume KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS ManagedBootImagesAzure ManagedBootImagesvSphere MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages PreconfiguredUDNAddresses ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SigstoreImageVerification SigstoreImageVerificationPKI StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSDualStackInstall AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureDualStackInstall AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement CBORServingAndStorage CRDCompatibilityRequirementOperator ClientsAllowCBOR ClientsPreferCBOR ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterAPIMachineManagement ClusterAPIMachineManagementVSphere ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud EtcdBackendQuota EventTTL EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall GCPDualStackInstall ImageModeStatusReporting IngressControllerDynamicConfigurationManager InsightsConfig InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryption KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesCPMS MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutableCSINodeAllocatableCount MutatingAdmissionPolicy NewOLM NewOLMBoxCutterRuntime NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterInstall NutanixMultiSubnets OSStreams OVNObservability OnPremDNSRecords OpenShiftPodSecurityAdmission ProvisioningRequestAvailable SELinuxMount ShortCertRotation SignatureStores TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeGroupSnapshot] I0504 20:45:11.850360 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0504 20:45:11.850376 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0504 20:45:11.850377 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0504 20:45:11.850385 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0504 20:45:11.850397 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0504 20:45:11.850402 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0504 20:45:11.850734 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0504 20:45:11.851214 1 secure_serving.go:213] Serving securely on [::]:8443 I0504 20:45:11.851249 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0504 20:45:11.857772 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0504 20:45:11.857812 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0504 20:45:11.857974 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0504 20:45:11.862364 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 20:45:11.862386 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0504 20:45:11.869436 1 secretconfigobserver.go:119] support secret does not exist I0504 20:45:11.874365 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 20:45:11.878893 1 secretconfigobserver.go:119] support secret does not exist I0504 20:45:11.883108 1 recorder.go:176] Pruning old reports every 4h36m54s, max age is 288h0m0s I0504 20:45:11.888676 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0504 20:45:11.888684 1 periodic.go:216] Running clusterconfig gatherer I0504 20:45:11.888698 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0504 20:45:11.888720 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0504 20:45:11.888721 1 tasks_processing.go:45] number of workers: 64 I0504 20:45:11.888725 1 insightsreport.go:296] Starting report retriever I0504 20:45:11.888729 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0504 20:45:11.888740 1 tasks_processing.go:69] worker 2 listening for tasks. I0504 20:45:11.888746 1 tasks_processing.go:69] worker 1 listening for tasks. I0504 20:45:11.888745 1 tasks_processing.go:69] worker 0 listening for tasks. I0504 20:45:11.888752 1 tasks_processing.go:69] worker 3 listening for tasks. I0504 20:45:11.888754 1 tasks_processing.go:69] worker 5 listening for tasks. I0504 20:45:11.888761 1 tasks_processing.go:69] worker 4 listening for tasks. I0504 20:45:11.888764 1 tasks_processing.go:69] worker 10 listening for tasks. I0504 20:45:11.888769 1 tasks_processing.go:69] worker 7 listening for tasks. I0504 20:45:11.888772 1 tasks_processing.go:69] worker 33 listening for tasks. I0504 20:45:11.888769 1 tasks_processing.go:69] worker 16 listening for tasks. I0504 20:45:11.888778 1 tasks_processing.go:71] worker 33 working on jaegers task. I0504 20:45:11.888777 1 tasks_processing.go:69] worker 6 listening for tasks. I0504 20:45:11.888800 1 tasks_processing.go:69] worker 47 listening for tasks. I0504 20:45:11.888775 1 tasks_processing.go:69] worker 8 listening for tasks. I0504 20:45:11.888802 1 tasks_processing.go:69] worker 9 listening for tasks. I0504 20:45:11.888776 1 tasks_processing.go:71] worker 7 working on olm_operators task. I0504 20:45:11.888814 1 tasks_processing.go:69] worker 61 listening for tasks. I0504 20:45:11.888815 1 tasks_processing.go:69] worker 39 listening for tasks. I0504 20:45:11.888815 1 tasks_processing.go:69] worker 12 listening for tasks. I0504 20:45:11.888810 1 tasks_processing.go:69] worker 46 listening for tasks. I0504 20:45:11.888823 1 tasks_processing.go:69] worker 40 listening for tasks. I0504 20:45:11.888825 1 tasks_processing.go:69] worker 59 listening for tasks. I0504 20:45:11.888831 1 tasks_processing.go:69] worker 14 listening for tasks. I0504 20:45:11.888808 1 tasks_processing.go:69] worker 11 listening for tasks. I0504 20:45:11.888840 1 tasks_processing.go:71] worker 1 working on service_accounts task. I0504 20:45:11.888840 1 tasks_processing.go:69] worker 45 listening for tasks. I0504 20:45:11.888841 1 tasks_processing.go:69] worker 36 listening for tasks. I0504 20:45:11.888847 1 tasks_processing.go:69] worker 15 listening for tasks. I0504 20:45:11.888826 1 tasks_processing.go:69] worker 34 listening for tasks. I0504 20:45:11.888829 1 tasks_processing.go:69] worker 62 listening for tasks. I0504 20:45:11.888833 1 tasks_processing.go:69] worker 63 listening for tasks. I0504 20:45:11.888855 1 tasks_processing.go:71] worker 0 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0504 20:45:11.888857 1 tasks_processing.go:69] worker 38 listening for tasks. I0504 20:45:11.888862 1 tasks_processing.go:71] worker 3 working on install_plans task. I0504 20:45:11.888864 1 tasks_processing.go:71] worker 4 working on nodes task. I0504 20:45:11.888834 1 tasks_processing.go:69] worker 44 listening for tasks. I0504 20:45:11.888872 1 tasks_processing.go:69] worker 52 listening for tasks. I0504 20:45:11.888836 1 tasks_processing.go:71] worker 2 working on operators task. I0504 20:45:11.888881 1 tasks_processing.go:69] worker 28 listening for tasks. I0504 20:45:11.888880 1 tasks_processing.go:69] worker 42 listening for tasks. I0504 20:45:11.888878 1 tasks_processing.go:69] worker 24 listening for tasks. I0504 20:45:11.888847 1 tasks_processing.go:69] worker 49 listening for tasks. I0504 20:45:11.888898 1 tasks_processing.go:69] worker 26 listening for tasks. I0504 20:45:11.888901 1 tasks_processing.go:69] worker 18 listening for tasks. I0504 20:45:11.888849 1 tasks_processing.go:69] worker 37 listening for tasks. I0504 20:45:11.888895 1 tasks_processing.go:71] worker 10 working on validating_webhook_configurations task. I0504 20:45:11.888907 1 tasks_processing.go:69] worker 27 listening for tasks. I0504 20:45:11.888856 1 tasks_processing.go:69] worker 50 listening for tasks. I0504 20:45:11.888853 1 tasks_processing.go:71] worker 5 working on image_registries task. I0504 20:45:11.888926 1 tasks_processing.go:69] worker 23 listening for tasks. I0504 20:45:11.888927 1 tasks_processing.go:69] worker 29 listening for tasks. I0504 20:45:11.888935 1 tasks_processing.go:69] worker 53 listening for tasks. I0504 20:45:11.888932 1 tasks_processing.go:69] worker 21 listening for tasks. I0504 20:45:11.888939 1 tasks_processing.go:69] worker 20 listening for tasks. I0504 20:45:11.888944 1 tasks_processing.go:69] worker 31 listening for tasks. I0504 20:45:11.888952 1 tasks_processing.go:69] worker 56 listening for tasks. I0504 20:45:11.888954 1 tasks_processing.go:69] worker 58 listening for tasks. I0504 20:45:11.888953 1 tasks_processing.go:69] worker 57 listening for tasks. I0504 20:45:11.888850 1 tasks_processing.go:69] worker 43 listening for tasks. I0504 20:45:11.888961 1 tasks_processing.go:69] worker 54 listening for tasks. I0504 20:45:11.888963 1 tasks_processing.go:69] worker 32 listening for tasks. I0504 20:45:11.888866 1 tasks_processing.go:69] worker 51 listening for tasks. I0504 20:45:11.888961 1 tasks_processing.go:69] worker 55 listening for tasks. I0504 20:45:11.888870 1 tasks_processing.go:69] worker 41 listening for tasks. I0504 20:45:11.888833 1 tasks_processing.go:69] worker 35 listening for tasks. I0504 20:45:11.888892 1 tasks_processing.go:69] worker 25 listening for tasks. I0504 20:45:11.888890 1 tasks_processing.go:69] worker 17 listening for tasks. I0504 20:45:11.888911 1 tasks_processing.go:69] worker 19 listening for tasks. I0504 20:45:11.888917 1 tasks_processing.go:69] worker 30 listening for tasks. I0504 20:45:11.888919 1 tasks_processing.go:69] worker 22 listening for tasks. I0504 20:45:11.888840 1 tasks_processing.go:69] worker 48 listening for tasks. I0504 20:45:11.888808 1 tasks_processing.go:69] worker 60 listening for tasks. I0504 20:45:11.889007 1 tasks_processing.go:71] worker 36 working on silenced_alerts task. I0504 20:45:11.889010 1 tasks_processing.go:71] worker 27 working on overlapping_namespace_uids task. I0504 20:45:11.888824 1 tasks_processing.go:69] worker 13 listening for tasks. I0504 20:45:11.889082 1 tasks_processing.go:71] worker 44 working on openstack_dataplanedeployments task. I0504 20:45:11.889099 1 tasks_processing.go:71] worker 39 working on node_features task. I0504 20:45:11.889120 1 tasks_processing.go:71] worker 47 working on storage_cluster task. I0504 20:45:11.889148 1 tasks_processing.go:71] worker 18 working on monitoring_persistent_volumes task. I0504 20:45:11.889056 1 tasks_processing.go:71] worker 52 working on proxies task. I0504 20:45:11.889172 1 tasks_processing.go:71] worker 16 working on container_images task. I0504 20:45:11.889200 1 tasks_processing.go:71] worker 26 working on networks task. I0504 20:45:11.889218 1 tasks_processing.go:71] worker 54 working on certificate_signing_requests task. I0504 20:45:11.889282 1 tasks_processing.go:71] worker 37 working on aggregated_monitoring_cr_names task. I0504 20:45:11.889344 1 tasks_processing.go:71] worker 25 working on dvo_metrics task. I0504 20:45:11.889405 1 tasks_processing.go:71] worker 42 working on openshift_machine_api_events task. I0504 20:45:11.889477 1 tasks_processing.go:71] worker 28 working on openstack_controlplanes task. I0504 20:45:11.889541 1 tasks_processing.go:71] worker 60 working on sap_datahubs task. I0504 20:45:11.889581 1 tasks_processing.go:71] worker 6 working on image task. I0504 20:45:11.889749 1 tasks_processing.go:71] worker 24 working on ingress task. I0504 20:45:11.889771 1 tasks_processing.go:71] worker 50 working on cluster_apiserver task. I0504 20:45:11.889808 1 tasks_processing.go:71] worker 30 working on sap_config task. I0504 20:45:11.889856 1 tasks_processing.go:71] worker 20 working on active_alerts task. I0504 20:45:11.889062 1 tasks_processing.go:71] worker 15 working on metrics task. I0504 20:45:11.890084 1 tasks_processing.go:71] worker 49 working on openstack_version task. I0504 20:45:11.890224 1 tasks_processing.go:71] worker 23 working on version task. I0504 20:45:11.890260 1 tasks_processing.go:71] worker 31 working on config_maps task. I0504 20:45:11.890289 1 tasks_processing.go:71] worker 22 working on schedulers task. I0504 20:45:11.889068 1 tasks_processing.go:71] worker 34 working on mutating_webhook_configurations task. I0504 20:45:11.890385 1 tasks_processing.go:71] worker 61 working on infrastructures task. I0504 20:45:11.889072 1 tasks_processing.go:71] worker 62 working on machines task. I0504 20:45:11.890561 1 tasks_processing.go:71] worker 48 working on storage_classes task. I0504 20:45:11.890697 1 tasks_processing.go:71] worker 14 working on qemu_kubevirt_launcher_logs task. I0504 20:45:11.890731 1 tasks_processing.go:71] worker 29 working on authentication task. I0504 20:45:11.890814 1 tasks_processing.go:71] worker 56 working on pod_network_connectivity_checks task. I0504 20:45:11.890861 1 tasks_processing.go:71] worker 53 working on crds task. I0504 20:45:11.889075 1 tasks_processing.go:71] worker 63 working on nodenetworkconfigurationpolicies task. I0504 20:45:11.890898 1 tasks_processing.go:71] worker 21 working on feature_gates task. I0504 20:45:11.890904 1 tasks_processing.go:71] worker 59 working on cost_management_metrics_configs task. I0504 20:45:11.890957 1 tasks_processing.go:71] worker 58 working on operators_pods_and_events task. I0504 20:45:11.891021 1 tasks_processing.go:71] worker 57 working on machine_config_pools task. I0504 20:45:11.891031 1 tasks_processing.go:71] worker 43 working on ingress_certificates task. I0504 20:45:11.891037 1 tasks_processing.go:71] worker 46 working on machine_healthchecks task. I0504 20:45:11.889078 1 tasks_processing.go:71] worker 38 working on lokistack task. I0504 20:45:11.891113 1 tasks_processing.go:71] worker 40 working on tsdb_status task. I0504 20:45:11.891184 1 tasks_processing.go:71] worker 12 working on sap_pods task. I0504 20:45:11.889092 1 tasks_processing.go:71] worker 13 working on pdbs task. I0504 20:45:11.891201 1 tasks_processing.go:71] worker 55 working on machine_sets task. I0504 20:45:11.891237 1 tasks_processing.go:71] worker 41 working on openstack_dataplanenodesets task. I0504 20:45:11.891328 1 tasks_processing.go:71] worker 35 working on container_runtime_configs task. I0504 20:45:11.891338 1 tasks_processing.go:71] worker 32 working on openshift_logging task. I0504 20:45:11.891341 1 tasks_processing.go:71] worker 51 working on machine_configs task. I0504 20:45:11.891021 1 tasks_processing.go:71] worker 45 working on clusterroles task. I0504 20:45:11.890349 1 tasks_processing.go:71] worker 17 working on nodenetworkstates task. I0504 20:45:11.890373 1 tasks_processing.go:71] worker 8 working on node_logs task. I0504 20:45:11.890379 1 tasks_processing.go:71] worker 9 working on oauths task. I0504 20:45:11.891026 1 tasks_processing.go:71] worker 11 working on ceph_cluster task. I0504 20:45:11.890354 1 tasks_processing.go:71] worker 19 working on image_pruners task. I0504 20:45:11.894301 1 tasks_processing.go:71] worker 33 working on support_secret task. I0504 20:45:11.894316 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 5.511024ms to process 0 records E0504 20:45:11.896732 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.219.31:9091: connect: connection refused I0504 20:45:11.896760 1 tasks_processing.go:71] worker 40 working on machine_autoscalers task. E0504 20:45:11.896806 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.219.31:9091: connect: connection refused I0504 20:45:11.896879 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 5.625076ms to process 0 records E0504 20:45:11.896840 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.219.31:9091: connect: connection refused I0504 20:45:11.896900 1 tasks_processing.go:74] worker 15 stopped. E0504 20:45:11.896908 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.219.31:9091: connect: connection refused I0504 20:45:11.896930 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 6.901951ms to process 0 records I0504 20:45:11.915948 1 tasks_processing.go:74] worker 60 stopped. I0504 20:45:11.915962 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 26.382503ms to process 0 records I0504 20:45:11.919584 1 tasks_processing.go:74] worker 30 stopped. I0504 20:45:11.919598 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 29.765834ms to process 0 records I0504 20:45:11.921738 1 tasks_processing.go:74] worker 42 stopped. I0504 20:45:11.921754 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 32.311756ms to process 0 records I0504 20:45:11.921822 1 tasks_processing.go:74] worker 28 stopped. I0504 20:45:11.921841 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 32.317616ms to process 0 records E0504 20:45:11.921872 1 gather_node_features.go:86] GatherNodeFeatures: NodeFeatures resource not found in openshift-nfd namespace (may not be installed) I0504 20:45:11.921893 1 tasks_processing.go:74] worker 39 stopped. I0504 20:45:11.921903 1 gather.go:177] gatherer "clusterconfig" function "node_features" took 32.78462ms to process 0 records I0504 20:45:11.921964 1 tasks_processing.go:74] worker 6 stopped. I0504 20:45:11.922121 1 recorder.go:75] Recording config/image with fingerprint=733b42d8674ab5af7c084d026764c25fcac8eda1b7433e2ea2be2d9ecf483d9d I0504 20:45:11.922146 1 gather.go:177] gatherer "clusterconfig" function "image" took 32.363673ms to process 1 records E0504 20:45:11.922157 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0504 20:45:11.922164 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 31.185072ms to process 0 records I0504 20:45:11.922192 1 tasks_processing.go:74] worker 56 stopped. I0504 20:45:11.922225 1 tasks_processing.go:74] worker 50 stopped. I0504 20:45:11.922326 1 controller.go:129] Initializing last reported time to 0001-01-01T00:00:00Z I0504 20:45:11.922331 1 recorder.go:75] Recording config/apiserver with fingerprint=5665eb2f7434225a77010c386c4caa75fedf8add06f8988b3de92783fe4c304e I0504 20:45:11.922351 1 controller.go:368] The initial operator extension status is healthy I0504 20:45:11.922351 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 32.234988ms to process 1 records I0504 20:45:11.922367 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0504 20:45:11.922380 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0504 20:45:11.922386 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0504 20:45:11.922409 1 controller.go:531] The operator is still being initialized I0504 20:45:11.922420 1 controller.go:554] The operator is healthy I0504 20:45:11.922429 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=ccd6bc7596c00e83b76fd2df49a2120010cb0436b16027dc037fd785d965468c I0504 20:45:11.922437 1 tasks_processing.go:74] worker 48 stopped. I0504 20:45:11.922444 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=3e8da2baf65cfdb99d2d1c03750f4ec3175e4c8dbe7038210f26d0ace6167481 I0504 20:45:11.922460 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 31.47426ms to process 2 records I0504 20:45:11.922571 1 tasks_processing.go:74] worker 26 stopped. I0504 20:45:11.922591 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0504 20:45:11.922601 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s I0504 20:45:11.922654 1 recorder.go:75] Recording config/network with fingerprint=8c7237dc826c6a6a89dc41b3c7e1517457dc527d401122c93e17adf34cf5c17d I0504 20:45:11.922710 1 gather.go:177] gatherer "clusterconfig" function "networks" took 32.959714ms to process 1 records W0504 20:45:11.922670 1 operator.go:328] started I0504 20:45:11.922690 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0504 20:45:11.922726 1 tasks_processing.go:74] worker 49 stopped. I0504 20:45:11.922721 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 32.085399ms to process 0 records I0504 20:45:11.922817 1 tasks_processing.go:74] worker 24 stopped. I0504 20:45:11.922908 1 recorder.go:75] Recording config/ingress with fingerprint=fba80a740b3a6f6a620f62416ebbb0024618f9b9535549511bd698629572573d I0504 20:45:11.922922 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 32.49073ms to process 1 records E0504 20:45:11.922929 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0504 20:45:11.922934 1 gather.go:177] gatherer "clusterconfig" function "machines" took 31.731281ms to process 0 records I0504 20:45:11.922936 1 tasks_processing.go:74] worker 62 stopped. E0504 20:45:11.922940 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0504 20:45:11.922944 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 31.397977ms to process 0 records I0504 20:45:11.922950 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 31.465521ms to process 0 records I0504 20:45:11.922953 1 tasks_processing.go:74] worker 46 stopped. I0504 20:45:11.922955 1 tasks_processing.go:74] worker 38 stopped. W0504 20:45:11.924195 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0504 20:45:11.924680 1 tasks_processing.go:74] worker 44 stopped. I0504 20:45:11.924697 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 35.574092ms to process 0 records I0504 20:45:11.924860 1 tasks_processing.go:74] worker 55 stopped. I0504 20:45:11.924869 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 33.651551ms to process 0 records I0504 20:45:11.924881 1 tasks_processing.go:74] worker 59 stopped. I0504 20:45:11.924896 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 33.959942ms to process 0 records I0504 20:45:11.924913 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 33.636556ms to process 0 records I0504 20:45:11.924921 1 tasks_processing.go:74] worker 41 stopped. I0504 20:45:11.925057 1 tasks_processing.go:74] worker 63 stopped. I0504 20:45:11.925068 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 34.158393ms to process 0 records I0504 20:45:11.925413 1 tasks_processing.go:74] worker 7 stopped. I0504 20:45:11.925432 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 36.589731ms to process 0 records I0504 20:45:11.925502 1 tasks_processing.go:74] worker 29 stopped. I0504 20:45:11.925685 1 recorder.go:75] Recording config/authentication with fingerprint=a3e81e7626316dcd0de2253b2c72cea641537cbf1e6d6a1b7e85130221de5c2a I0504 20:45:11.925706 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 34.751369ms to process 1 records I0504 20:45:11.925777 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=0908c8e536d9029a86e316b71b1a0def33d5d2af0251475f4edb09833af51fd2 I0504 20:45:11.925796 1 tasks_processing.go:74] worker 22 stopped. I0504 20:45:11.925805 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 35.224841ms to process 1 records I0504 20:45:11.925896 1 tasks_processing.go:74] worker 61 stopped. I0504 20:45:11.926670 1 recorder.go:75] Recording config/infrastructure with fingerprint=74162fea8642f3de70810a653a4d197ff4ac888cd39871cd0534ff7bfff434cc I0504 20:45:11.926865 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 35.358841ms to process 1 records I0504 20:45:11.926901 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 34.584458ms to process 0 records I0504 20:45:11.926930 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 34.607635ms to process 0 records I0504 20:45:11.926967 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 34.446309ms to process 0 records I0504 20:45:11.926992 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 36.837878ms to process 0 records I0504 20:45:11.927016 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 35.076971ms to process 0 records I0504 20:45:11.927039 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 35.273109ms to process 0 records I0504 20:45:11.927105 1 tasks_processing.go:74] worker 57 stopped. I0504 20:45:11.927129 1 tasks_processing.go:74] worker 47 stopped. I0504 20:45:11.927183 1 tasks_processing.go:74] worker 12 stopped. I0504 20:45:11.927099 1 tasks_processing.go:74] worker 35 stopped. I0504 20:45:11.927732 1 tasks_processing.go:74] worker 17 stopped. I0504 20:45:11.928046 1 tasks_processing.go:74] worker 34 stopped. I0504 20:45:11.927661 1 tasks_processing.go:74] worker 32 stopped. I0504 20:45:11.930323 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=8e9fc81bab18bd05a56bd1fc9becf0e90b660da9b902e957d6784e6443bebd45 I0504 20:45:11.930384 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=a9f73a6f32549e65873a78e120b6865243917ea14221d7c31dde306a47c8ca68 I0504 20:45:11.930422 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=f71ddecaf8e4363a5b6b2063fd23f7f7e918280f1d0d5069f33d8bd8f34b450a I0504 20:45:11.930433 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 37.582893ms to process 3 records I0504 20:45:11.930522 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=3a9dd324d4f5e28d74f1dac87d1148af25c81f0348f00cbd04dc641d057a666a I0504 20:45:11.930550 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=68da40beba78d40eb924f370b07c76be07449c96c91c97855ae36d316b9b6bfa I0504 20:45:11.930614 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=6828d0231242ca22372eb725aea65fb20b811c1411c01646306286bc1160aadc I0504 20:45:11.930628 1 tasks_processing.go:74] worker 13 stopped. I0504 20:45:11.930645 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=7d3872516968a671ec43d214cdccfe61a00ae9bfe0cef97c83eb9e7621a5bf2f I0504 20:45:11.930669 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=25563f9cc08511d46baba8c3451cd8a0c0a912039e4f4a262c02bf1ba923a231 I0504 20:45:11.930691 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=f8a042a94d52b6a60cfc34f991c4134db8c2102b8eb839e65c8ef33d390738fe I0504 20:45:11.930710 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=67054b7c39d0b81f1600047beca5dd13376d0f28d3f4c37df68c889a67cbd41e I0504 20:45:11.930736 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=74c2e10532679b133d9a99da827258728c2872cac3d6d1cc8db335a030d661f5 I0504 20:45:11.930804 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=84d2333888dff8626d402b6b2476826cfe57ae8649ac94f725956de18f8c8772 I0504 20:45:11.930860 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=ff6813558bb434d293af6005900838b2921c0ca3ab76fd6440cb23f04770c442 I0504 20:45:11.930904 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=3e4713b82d29b5e89fcdb656bb117e5680eede865b4fe63e95aad0f0dfec89e6 I0504 20:45:11.930917 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 37.707975ms to process 11 records I0504 20:45:11.930929 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 32.452077ms to process 0 records I0504 20:45:11.930951 1 tasks_processing.go:74] worker 40 stopped. I0504 20:45:11.931028 1 tasks_processing.go:74] worker 10 stopped. I0504 20:45:11.931044 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=916cfad389ccb9ddd01469a3b5b3d1229a5cf1a8180c299f0cdd3015a9c2f59e I0504 20:45:11.931100 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=937dd1e6cb1c8f6a9bc582ea3e68c3da97905e83fc068baaac00373c960d56a0 I0504 20:45:11.931200 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=7aa035c2511349aa8f77b0bc5f60c98b26dd792e3209a70365f55caa80a833ef I0504 20:45:11.931236 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=a1e5c1e32befef17bf02efcdb1d5f114b9fcc27d91921d6171122bc03c0bef08 I0504 20:45:11.931267 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=965cc9e53fa7bdbedfc87b911172dfe82f58f060fc45a034604532af1d1bcf13 I0504 20:45:11.931292 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=78eb2c47af34759b12e0abf90208893e34e51bd728fb52f923b973c53c0b806f I0504 20:45:11.931330 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=eb51265edd12165e8fe352a4ee4974ae11b7519c3568b73e3efc627039d0f8cf I0504 20:45:11.931354 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=b9d2bb29d73ee23fcb4d557828e5c280a0f31fc726b6a55b73121eb973f91e63 I0504 20:45:11.931414 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=13859fc2155b743802423c0ded93706968c5cfa2973edb3050724ec70d78ab7e I0504 20:45:11.931471 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=a343777c8932eaa23f866c89dd4847f8becb8efd80076a18ded972c6f4745c12 I0504 20:45:11.931510 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=98ed067eb210cef60274ab6d7d68aba17261b90234ef6fffdd536a9cc03281dc I0504 20:45:11.931579 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=60817ccae1f0d37ca3f455709a7ea486193955fc8fb03ab8fe6d82da094b3e6e I0504 20:45:11.931622 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=1daee0c0de6ace3cb62992477ea6847ce0df02cfb8897c9c5cda8f198541d314 I0504 20:45:11.931633 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 40.426636ms to process 13 records I0504 20:45:11.931645 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 40.248542ms to process 0 records I0504 20:45:11.931651 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 37.258548ms to process 0 records I0504 20:45:11.931664 1 tasks_processing.go:74] worker 18 stopped. I0504 20:45:11.931673 1 tasks_processing.go:74] worker 11 stopped. I0504 20:45:11.931736 1 tasks_processing.go:74] worker 21 stopped. I0504 20:45:11.931811 1 recorder.go:75] Recording config/featuregate with fingerprint=0a9a38d8da8404cf675a9465dc61d1e7b98d60847092759c0098d79c782a8bae I0504 20:45:11.931827 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 38.77312ms to process 1 records I0504 20:45:11.931849 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0504 20:45:11.931856 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 41.930274ms to process 1 records I0504 20:45:11.931921 1 tasks_processing.go:74] worker 27 stopped. I0504 20:45:11.931996 1 tasks_processing.go:74] worker 4 stopped. I0504 20:45:11.932367 1 recorder.go:75] Recording config/node/ip-10-0-0-19.ec2.internal with fingerprint=d3ced407f207a8c078fb11e0296590acd9d3613c10eb79937e3a9cc6fa68a2f4 I0504 20:45:11.932493 1 recorder.go:75] Recording config/node/ip-10-0-1-83.ec2.internal with fingerprint=7ad325302d69c2b86a7822de44a9b5cd4f9e01358fe8c683564f60b31e5978be I0504 20:45:11.932628 1 recorder.go:75] Recording config/node/ip-10-0-2-46.ec2.internal with fingerprint=0a29932910ca4150e03af019ab3d239e13ab14402d8672e448152f7be516c9d8 I0504 20:45:11.932644 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 42.147448ms to process 3 records I0504 20:45:11.932664 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 42.789678ms to process 0 records I0504 20:45:11.932680 1 tasks_processing.go:74] worker 54 stopped. I0504 20:45:11.933864 1 tasks_processing.go:74] worker 58 stopped. I0504 20:45:11.933878 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 42.886397ms to process 0 records I0504 20:45:11.934260 1 tasks_processing.go:74] worker 52 stopped. I0504 20:45:11.934316 1 recorder.go:75] Recording config/proxy with fingerprint=19581c64271f7d57bd7b19ea11d5ea9e11adc3668212b7da3ab4f989446c9986 I0504 20:45:11.934330 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 45.089087ms to process 1 records I0504 20:45:11.934389 1 tasks_processing.go:74] worker 9 stopped. I0504 20:45:11.934572 1 recorder.go:75] Recording config/oauth with fingerprint=94a834111479e52c8a24844c960abff58aa391c2d5da5c9c6ae95e4ddd543fb3 I0504 20:45:11.934584 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 42.312351ms to process 1 records I0504 20:45:11.951166 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0504 20:45:11.951183 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0504 20:45:11.951254 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0504 20:45:11.958458 1 base_controller.go:82] Caches are synced for ConfigController I0504 20:45:11.958471 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0504 20:45:11.959593 1 tasks_processing.go:74] worker 33 stopped. E0504 20:45:11.959722 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0504 20:45:11.959773 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 65.274126ms to process 0 records E0504 20:45:11.959836 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0504 20:45:11.959942 1 tasks_processing.go:74] worker 51 stopped. I0504 20:45:11.959951 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0504 20:45:11.960018 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 68.354002ms to process 1 records I0504 20:45:11.961611 1 tasks_processing.go:74] worker 19 stopped. I0504 20:45:11.962051 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=195c558188da542ddd0f79911bf03e41e83af01faafa29f0147fb9da71a29886 I0504 20:45:11.962070 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 69.310925ms to process 1 records I0504 20:45:11.962530 1 tasks_processing.go:74] worker 45 stopped. I0504 20:45:11.962701 1 controller.go:254] Source scaController *sca.Controller is not ready I0504 20:45:11.962718 1 controller.go:254] Source clusterTransferController *clustertransfer.Controller is not ready I0504 20:45:11.962724 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0504 20:45:11.962728 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0504 20:45:11.962732 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0504 20:45:11.962736 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=2266a0c32448c6cc1e3417e80d37d9c8d8ff810fdcaa1f1f6a75f82149cbf345 I0504 20:45:11.962755 1 controller.go:531] The operator is still being initialized I0504 20:45:11.962764 1 controller.go:554] The operator is healthy I0504 20:45:11.962876 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=a84d042860b7e86d40608e6dca7d5e1218409552b00d842a319dcc34656e6455 I0504 20:45:11.962892 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 71.081555ms to process 2 records I0504 20:45:11.962960 1 tasks_processing.go:74] worker 53 stopped. I0504 20:45:11.963753 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=dcaab0b72c78beae224908e7c950c7ba9e68441f18c02cef1da71a4cfde8d8b0 I0504 20:45:11.964148 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=0428f6733efca4f5f6daae44a296983265559f6ef812e68e0b35e63902e51733 I0504 20:45:11.964165 1 gather.go:177] gatherer "clusterconfig" function "crds" took 72.077675ms to process 2 records I0504 20:45:11.964179 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 73.924143ms to process 0 records I0504 20:45:11.964200 1 tasks_processing.go:74] worker 37 stopped. I0504 20:45:11.964267 1 tasks_processing.go:74] worker 5 stopped. I0504 20:45:11.964526 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=b0a3d792560affbd3136b024d1385a361625cf74a2a48266a76700d170760d9f I0504 20:45:11.964538 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 74.52636ms to process 1 records I0504 20:45:11.970730 1 configmapobserver.go:84] configmaps "insights-config" not found I0504 20:45:11.989363 1 tasks_processing.go:74] worker 0 stopped. I0504 20:45:11.989386 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 100.487205ms to process 0 records I0504 20:45:11.989996 1 tasks_processing.go:74] worker 16 stopped. I0504 20:45:11.991356 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-66dbbb6468-vb597 with fingerprint=7edf6f48761dc76773459a50ce1014a85485b78cd7f82687bc7409e509e154a3 I0504 20:45:11.991727 1 recorder.go:75] Recording config/pod/openshift-ovn-kubernetes/ovnkube-node-pkxgm with fingerprint=846f79595ad1969c35a6e3195d0cb553b0f20d9e23b220cb89be3b8c4b7ee361 I0504 20:45:11.992006 1 recorder.go:75] Recording config/pod/openshift-ovn-kubernetes/ovnkube-node-xm4vg with fingerprint=fdf9319dad2536be17497f87b1a296f070d4c8c0aedfa44782fd5cac845cf0cb I0504 20:45:11.992114 1 recorder.go:75] Recording config/running_containers with fingerprint=eaac9e38b123c50cfab52ff2ab0ea5780e26e7faebdd7377064c979788412aee I0504 20:45:11.992127 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 100.80797ms to process 4 records I0504 20:45:11.993236 1 tasks_processing.go:74] worker 31 stopped. E0504 20:45:11.993256 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0504 20:45:11.993269 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0504 20:45:11.993276 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0504 20:45:11.993291 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=ad286723f58bdcfc37aeba3ec5b4110c08e1af59cd34d14b4bfaab02d18e4856 I0504 20:45:11.993349 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0504 20:45:11.993358 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0504 20:45:11.993382 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=2ab5ab7b1b10d7fcf1197bb24dea7c90f400e4effc18c7356873209d54fdf84b I0504 20:45:11.993402 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0504 20:45:11.993467 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0504 20:45:11.993478 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0504 20:45:11.993486 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 102.96126ms to process 7 records I0504 20:45:11.997544 1 tasks_processing.go:74] worker 43 stopped. E0504 20:45:11.997561 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0504 20:45:11.997569 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2q3b4g5ismrf3h31o672c5ttvbsfonn0-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2q3b4g5ismrf3h31o672c5ttvbsfonn0-primary-cert-bundle-secret" not found I0504 20:45:11.997646 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=a8e36869bde2ddb6fef7bf3860443c8313a572dfcfe3d101434fd5364397d3cb I0504 20:45:11.997674 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 106.494827ms to process 1 records I0504 20:45:12.008723 1 tasks_processing.go:74] worker 23 stopped. I0504 20:45:12.008973 1 recorder.go:75] Recording config/version with fingerprint=ab444211634da2605bc888ef25b3ec2090993ba6b26a2ca2b54cb7ec53ea6bf7 I0504 20:45:12.008985 1 recorder.go:75] Recording config/id with fingerprint=e6140ea8a09ab9758f3980554b7c762fce0a5233d29d7ce947155afdb65ddb4e I0504 20:45:12.008991 1 gather.go:177] gatherer "clusterconfig" function "version" took 118.482193ms to process 2 records I0504 20:45:12.008997 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 117.23598ms to process 0 records I0504 20:45:12.009003 1 tasks_processing.go:74] worker 8 stopped. I0504 20:45:12.009126 1 gather_logs.go:145] no pods in namespace were found I0504 20:45:12.009150 1 tasks_processing.go:74] worker 14 stopped. I0504 20:45:12.009163 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 118.433342ms to process 0 records I0504 20:45:12.023130 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0504 20:45:12.023169 1 base_controller.go:82] Caches are synced for LoggingSyncer I0504 20:45:12.023177 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0504 20:45:12.041492 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0504 20:45:12.045621 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0504 20:45:12.049883 1 cluster_transfer.go:102] no available accepted cluster transfer I0504 20:45:12.049905 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0504 20:45:12.304205 1 sca.go:266] etc-pki-entitlement secret successfully created I0504 20:45:12.304222 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated W0504 20:45:12.925738 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0504 20:45:12.942042 1 tasks_processing.go:74] worker 36 stopped. I0504 20:45:12.942073 1 recorder.go:75] Recording config/silenced_alerts.json with fingerprint=37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570 I0504 20:45:12.942094 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 1.053016492s to process 1 records I0504 20:45:12.943115 1 tasks_processing.go:74] worker 20 stopped. I0504 20:45:12.943151 1 recorder.go:75] Recording config/alerts with fingerprint=69d84ab6f5f84091ef5cc1d7d2d7ef04d48459a1f8225728c93ea1e014ca7ef5 I0504 20:45:12.943163 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 1.053240064s to process 1 records W0504 20:45:13.924446 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0504 20:45:14.383951 1 tasks_processing.go:74] worker 2 stopped. I0504 20:45:14.384077 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=4a07806f3f1c590874d865e3d379174af10b6de9efacb15e4b135cb7fa8435f6 I0504 20:45:14.384136 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0504 20:45:14.384175 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=9107d4bdf67fe251d12e39cb09b47aba9d475b7fa408e6a3db5676e9c237b033 I0504 20:45:14.384187 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0504 20:45:14.384207 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=c5bb873fbc6339b1b23586c56ed74f1611aed5598b67a5adf9828573e5463112 I0504 20:45:14.384223 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0504 20:45:14.384244 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=053acdaacfca04a111ccfda611ed09e217967f1bb583abf98065293edf641aba I0504 20:45:14.384285 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=14ecbf05c07b1794cd477dfc1409f9637daf05f576549e762b5a25cefe2bde50 I0504 20:45:14.384312 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=b0a6c120c65d9bf650233976bcc0759404f012b7e0e6711b97659eca3812cc0a I0504 20:45:14.384343 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=0b82f2375f46b934e9d49b9f2a1c566a1aefc991c396129a15afc50fa79724c5 I0504 20:45:14.384710 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=bb61a4fba39260fc3df5d69824ab7133b9540b2512c69f387094f27205bba7a4 I0504 20:45:14.384740 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=ac568cf5c1481cb63959f8ec54dd77dcffc2dabc4a98545358f30eb51d347c82 I0504 20:45:14.384756 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0504 20:45:14.384802 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=2fab761b2b2b65efa677deaa7c60d518ce9473a1cdb9ddb9d053fb3d27e09f02 I0504 20:45:14.384815 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0504 20:45:14.384833 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=0f231ab92b5df51eddbc2533931fec0f8152d1cd80d42bd906a71bdb40e3f299 I0504 20:45:14.384842 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0504 20:45:14.384861 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=eeb6d8f01f92a70a1fd11f0184dd1d9d0e4b1dbcbaa14ff63e25db4a656ff6c5 I0504 20:45:14.384871 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0504 20:45:14.384890 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=7c2f72dcef06e7b075e1a4bd4f83ebd28d32faa89db0e6ee2c68c5ed815081ba I0504 20:45:14.385017 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=c0172e623b65566d4336232dbd40ad3a381bea3b35a87c47c71c421d1d2d59c4 I0504 20:45:14.385028 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0504 20:45:14.385034 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0504 20:45:14.385056 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0504 20:45:14.385082 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=8b26471c8bc3855ffab1be5287efd9cb42d6f3a7bd0c077da9f25e25e95025a7 I0504 20:45:14.385121 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=88c652774cb3a742ce8ef0f5e3e1337fe37294f83c88c2bd539ea09da440955d I0504 20:45:14.385134 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0504 20:45:14.385149 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=82c8c21e84ed48045c6ef4ac67f068b61bf26cb54c09b335cc9732be4c218506 I0504 20:45:14.385157 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0504 20:45:14.385178 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=f19b1a6374c43521ce9238ac60dd67b3d073f92695ec3d92b5f4a4ae34e938c5 I0504 20:45:14.385185 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0504 20:45:14.385198 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=d2527d7ac888ab621901bf14b6aacad24ab7e9be5227d8ffccc1fbbb062203d6 I0504 20:45:14.385212 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=79554bc41f300487a9df0cad862c390a1d9f852369b6cb92c8f98116e3ad229a I0504 20:45:14.385227 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=1ea150d324132e25720284a97aeba2de05546a1adfda899a1533b381bbba8218 I0504 20:45:14.385253 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=9ecff3fb1f37119065cc82872acdedf996af14e32f7553b2e0a8a657232db143 I0504 20:45:14.385267 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0504 20:45:14.385322 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=1677dc8f2c603e6dade6540e5fc4747d431e7ede8002b78b94371df93fc29a0d I0504 20:45:14.385340 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0504 20:45:14.385347 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0504 20:45:14.385354 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.495056212s to process 39 records W0504 20:45:14.924027 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0504 20:45:15.924023 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0504 20:45:16.924128 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0504 20:45:16.924163 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0504 20:45:16.924180 1 tasks_processing.go:74] worker 25 stopped. E0504 20:45:16.924189 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0504 20:45:16.924201 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0504 20:45:16.924211 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.03480082s to process 1 records I0504 20:45:24.736536 1 tasks_processing.go:74] worker 3 stopped. I0504 20:45:24.736578 1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b I0504 20:45:24.736595 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.847653065s to process 1 records I0504 20:45:25.495322 1 tasks_processing.go:74] worker 1 stopped. I0504 20:45:25.495586 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=80e5b054934e37ddf232a6b52185c88ef3149965af63f70b0125fd661e8f707e I0504 20:45:25.495603 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.606466371s to process 1 records E0504 20:45:25.495667 1 periodic.go:254] "Unhandled Error" err="clusterconfig failed after 13.606s with: function \"tsdb_status\" failed with an error, function \"metrics\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machines\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"support_secret\" failed with an error, function \"machine_configs\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0504 20:45:25.496774 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "tsdb_status" failed with an error, function "metrics" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machines" failed with an error, function "machine_healthchecks" failed with an error, function "support_secret" failed with an error, function "machine_configs" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0504 20:45:25.496808 1 periodic.go:216] Running workloads gatherer I0504 20:45:25.496825 1 tasks_processing.go:45] number of workers: 2 I0504 20:45:25.496839 1 tasks_processing.go:69] worker 1 listening for tasks. I0504 20:45:25.496843 1 tasks_processing.go:71] worker 1 working on workload_info task. I0504 20:45:25.496853 1 tasks_processing.go:69] worker 0 listening for tasks. I0504 20:45:25.496869 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0504 20:45:25.503391 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-19.ec2.internal... I0504 20:45:25.503401 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-46.ec2.internal... I0504 20:45:25.503403 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-83.ec2.internal... I0504 20:45:25.523198 1 tasks_processing.go:74] worker 0 stopped. I0504 20:45:25.523215 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 26.315774ms to process 0 records I0504 20:45:30.005671 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 5s I0504 20:45:30.031301 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0504 20:45:30.045204 1 gather_workloads_info.go:387] No image sha256:a043239802b3eb8b323d285193d2527fad0ecec98ca91d188a3472a2fac8ae04 (15ms) I0504 20:45:30.061231 1 gather_workloads_info.go:387] No image sha256:730d1b6988025bef0daa3a9a5d8467ec4a26b0382cc52f91c3375b4590d3518a (16ms) I0504 20:45:30.079226 1 gather_workloads_info.go:387] No image sha256:23204877f166455ce8f9de9b91a467eeda576f172d5f96124a68aa986c4775d6 (18ms) I0504 20:45:30.093104 1 gather_workloads_info.go:387] No image sha256:5d6a2b8d2508c7da71192661e7c0dcf1335f48b6336a20189abffbe79eeff962 (14ms) I0504 20:45:30.102624 1 gather_workloads_info.go:387] No image sha256:ae7d3453fd734ecc865e5f9bb16f29244ebffe6291b77e1d4e496f71eb053174 (10ms) I0504 20:45:30.112519 1 gather_workloads_info.go:387] No image sha256:5ac9c549d65fc1d8bc900773bebee43e9192bcec1bb5fa46afb4597230c16ac7 (10ms) I0504 20:45:30.121841 1 gather_workloads_info.go:387] No image sha256:000105ef5150e7079b90a613fb9e6193e2a6ef9b1908d2dce44f2395d4fd070f (9ms) I0504 20:45:30.130562 1 gather_workloads_info.go:387] No image sha256:15928a5b803fa00781b408dbc4c3bfa7c5287b18584ba31f817def8a4430c725 (9ms) I0504 20:45:30.139186 1 gather_workloads_info.go:387] No image sha256:0b2e6536a0ca3afc5e64a218087a6287c74b77a359075a385f1949ca5bd51b22 (9ms) I0504 20:45:30.147851 1 gather_workloads_info.go:387] No image sha256:ab60623bb32f7e75fca71ef65137731cae347a21c7a4091dfd583fa00732721c (9ms) I0504 20:45:30.156597 1 gather_workloads_info.go:387] No image sha256:1eb6a6ee3c66ac15be6bc21bb6015c595c11c15b9dfd21bd9f5ce7f7b990ebaa (9ms) I0504 20:45:30.240612 1 gather_workloads_info.go:387] No image sha256:52db05084f2a2e7d7bf4040e4865df88ec4074efbbe59a78d861be2c5ab2aa40 (84ms) I0504 20:45:30.340572 1 gather_workloads_info.go:387] No image sha256:d64bea34bf3e1bb0b3a701c3ff14e66665afc1b050f28124ad7e6888eaec3a81 (100ms) I0504 20:45:30.441094 1 gather_workloads_info.go:387] No image sha256:934d8e8c50f3c609b8eea80d1051111fe3d066fe8c65c79572072ae55fcb0a86 (101ms) I0504 20:45:30.540151 1 gather_workloads_info.go:387] No image sha256:53ff8efdd8f563421ebc1859ab021a3bbc1306f82d4ff5d9c21bc60ec5f57898 (99ms) I0504 20:45:30.640217 1 gather_workloads_info.go:387] No image sha256:6814f5fced0ef219d06374011c68a11a3da788a764a00a69ade435466d9ee240 (100ms) I0504 20:45:30.741267 1 gather_workloads_info.go:387] No image sha256:875c77e5d144f03fb91d8cee0259f6966683ca88d1bf818dbf4652c16b70312c (101ms) I0504 20:45:30.840032 1 gather_workloads_info.go:387] No image sha256:5ff204630794311b0b37fa7b197b933ad85d76a481bf7cdb3bcbada08f0cdcbf (99ms) I0504 20:45:30.940736 1 gather_workloads_info.go:387] No image sha256:5aaea0419169e55832cc27acfe0fe3b9513a343d6bdf71d3da1575ed322245d0 (101ms) I0504 20:45:31.041059 1 gather_workloads_info.go:387] No image sha256:0a4dfb8d4c1b3849319d45b4c54dff26a7238a2c08fcaa121f93073e95ab12e8 (100ms) I0504 20:45:31.140387 1 gather_workloads_info.go:387] No image sha256:08c5a78c8a5af04c549e2273aaf4bb452a75bf038d68aa9d01bb2aff66c30e90 (99ms) I0504 20:45:31.240673 1 gather_workloads_info.go:387] No image sha256:17aefdc2f4752c23dc8f0947bb01fc34635822eeb18f2d8a5eddf8211b84d450 (100ms) I0504 20:45:31.341612 1 gather_workloads_info.go:387] No image sha256:8f881f6f0498eaba860d711dc8e9dc066db8728f22abedb82e1dc9a8e1c30046 (101ms) I0504 20:45:31.439737 1 gather_workloads_info.go:387] No image sha256:15677f0b70e6aa2dfaf088e45fc1a425c22bf6fda326b8116f87e88b6694dfab (98ms) I0504 20:45:31.543108 1 gather_workloads_info.go:387] No image sha256:084aa9b0f8a6d478549dc384d4e66da13ee9b25cc98531da861cc19dee2a9e8f (103ms) I0504 20:45:31.639813 1 gather_workloads_info.go:387] No image sha256:bdeb1c455cffccdb6c90db238e9d371de06340b1d6105d47f66d87293f878ac1 (97ms) I0504 20:45:31.740996 1 gather_workloads_info.go:387] No image sha256:55b1db6038c5beaed54c626e3343b7a8589cc0be8dc41d1a66b4deab766ff520 (101ms) I0504 20:45:31.839770 1 gather_workloads_info.go:387] No image sha256:30e597ec5d6bb96ff70a4f8688c748b659cd4fd5d73d222e8701821d236795c5 (99ms) I0504 20:45:31.940751 1 gather_workloads_info.go:387] No image sha256:0acfe677ed58031f8381cb8a4f2d8917ec4cfa95485e2a46529dffae54c75ace (101ms) I0504 20:45:32.041002 1 gather_workloads_info.go:387] No image sha256:9d6d63dce784df5863e6a0ba4c11317a26c9d756d4e62d47759c97190a71cce5 (100ms) I0504 20:45:32.140957 1 gather_workloads_info.go:387] No image sha256:15e1b5f763553e86d0230a751b0783e3b990438fd07cf96eed164bccb72ed564 (100ms) I0504 20:45:32.240960 1 gather_workloads_info.go:387] No image sha256:5b6212b8f539f08e78417d8a4b7485ca0b4e7927cacd7b752742a28841bc8ccd (100ms) I0504 20:45:32.339983 1 gather_workloads_info.go:387] No image sha256:6c6d412ccf0c3d6a14645a79df6c4658c0f8725d7954cb83c3d2f0116499345a (99ms) I0504 20:45:32.440323 1 gather_workloads_info.go:387] No image sha256:5808401268394502d335281ea1a294b07210461b986b58f91d7d1f29c0029c6d (100ms) I0504 20:45:32.541580 1 gather_workloads_info.go:387] No image sha256:3380fecebaeee139c1607fbcd9346cebc92806679568e2244a8905ef0f52aad2 (101ms) I0504 20:45:32.640617 1 gather_workloads_info.go:387] No image sha256:3bba1358d4a0ae878ff491c0c2cbfffe60649e110b40342b878fe8fa332f8858 (99ms) I0504 20:45:32.741281 1 gather_workloads_info.go:387] No image sha256:572ebf7c7fab1a518582f3ca46c866730423cce6e8a19f614dc15fd61bf0c65a (101ms) I0504 20:45:32.839610 1 gather_workloads_info.go:387] No image sha256:5c6d21c3f97366bc7ab57031cc027b67405a684bd804ce364ed5998b0685eaca (98ms) I0504 20:45:32.940136 1 gather_workloads_info.go:387] No image sha256:5940cc34b322325618929b1e0eb3864dae73d54c8f5b474e218dfb28123a61c7 (101ms) I0504 20:45:33.039379 1 gather_workloads_info.go:387] No image sha256:9ef6dc311b880a6442b7a3a19006ddd955c7da36850bf000f5c9b68fd0862dd4 (99ms) I0504 20:45:33.139750 1 gather_workloads_info.go:387] No image sha256:637d41f067a5239096fc22b135181cda5113da833f1370e7a73965e83792e93a (100ms) I0504 20:45:33.240121 1 gather_workloads_info.go:387] No image sha256:48e883a3932aea9457f55cd4628d21397d429d4828f42f95d1c903d9d5395bde (100ms) I0504 20:45:33.340025 1 gather_workloads_info.go:387] No image sha256:673ebc8cc22c56c8f410e011b2fa950d28cf7b6420e17fdb6580d6cb10523384 (100ms) I0504 20:45:33.441713 1 gather_workloads_info.go:387] No image sha256:48f5559439bea030a6423652da448c25fff0c62ba293eb521843c1a8eb89ddb0 (102ms) I0504 20:45:33.540200 1 gather_workloads_info.go:387] No image sha256:25c148fd380b1a9db3f6039d2e0eabc489a954921452391390ba9192b2325678 (98ms) I0504 20:45:33.639770 1 gather_workloads_info.go:387] No image sha256:89277d8d4560d71db88c2dcc67c992a24544ca21810920b609c1d49d53b4a287 (100ms) I0504 20:45:33.747269 1 gather_workloads_info.go:387] No image sha256:1f118a4ec0b92cb88c8544ea31d2b9adfc7006eeddb7a8d260150ecbce4adb8c (107ms) I0504 20:45:33.747298 1 tasks_processing.go:74] worker 1 stopped. I0504 20:45:33.747581 1 recorder.go:75] Recording config/workload_info with fingerprint=f3e8cff40bb57006b44a9f583ee4ae2f194ad9ed2a6799d40052ac31c163948e I0504 20:45:33.747598 1 gather.go:177] gatherer "workloads" function "workload_info" took 8.250449108s to process 1 records I0504 20:45:33.747610 1 periodic.go:263] Periodic gather workloads completed in 8.25s I0504 20:45:33.747618 1 controllerstatus.go:80] name=periodic-workloads healthy=true reason= message= I0504 20:45:33.747623 1 periodic.go:216] Running conditional gatherer I0504 20:45:33.755036 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.21.9/gathering_rules I0504 20:45:33.913964 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer E0504 20:45:33.916352 1 conditional_gatherer.go:324] unable to update alerts cache: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?match%5B%5D=ALERTS%7Balertstate%3D%22firing%22%7D&query=ALERTS": dial tcp 172.30.219.31:9091: connect: connection refused I0504 20:45:33.916422 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0504 20:45:33.922304 1 conditional_gatherer.go:392] cluster version is '4.21.9' E0504 20:45:33.922319 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0504 20:45:33.922325 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0504 20:45:33.922328 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0504 20:45:33.922331 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0504 20:45:33.922334 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0504 20:45:33.922337 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0504 20:45:33.922340 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0504 20:45:33.922343 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0504 20:45:33.922346 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0504 20:45:33.922349 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0504 20:45:33.922363 1 tasks_processing.go:45] number of workers: 3 I0504 20:45:33.922377 1 tasks_processing.go:69] worker 2 listening for tasks. I0504 20:45:33.922381 1 tasks_processing.go:71] worker 2 working on remote_configuration task. I0504 20:45:33.922388 1 tasks_processing.go:69] worker 0 listening for tasks. I0504 20:45:33.922400 1 tasks_processing.go:71] worker 0 working on rapid_container_logs task. I0504 20:45:33.922410 1 tasks_processing.go:69] worker 1 listening for tasks. I0504 20:45:33.922424 1 tasks_processing.go:74] worker 1 stopped. I0504 20:45:33.922428 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0504 20:45:33.922443 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 884ns to process 1 records I0504 20:45:33.922435 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0504 20:45:33.922470 1 tasks_processing.go:74] worker 2 stopped. I0504 20:45:33.922579 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=9fafdcbf373f1f367f1d69551ea59114592cce1681189be0f3dde774e0530661 I0504 20:45:33.922594 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 5.969µs to process 1 records I0504 20:45:33.922864 1 tasks_processing.go:74] worker 0 stopped. I0504 20:45:33.922888 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 457.411µs to process 0 records I0504 20:45:33.922903 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0504 20:45:33.922918 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0504 20:45:33.949395 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=80bd65c2ba0edbfcbceef3442cecbee4e17067e260e9739142df5340765a237a I0504 20:45:33.949530 1 diskrecorder.go:70] Writing 112 records to /var/lib/insights-operator/insights-2026-05-04-204533.tar.gz I0504 20:45:33.956901 1 diskrecorder.go:51] Wrote 112 records to disk in 7ms I0504 20:45:33.956928 1 periodic.go:285] Gathering cluster info every 2h0m0s I0504 20:45:33.956942 1 periodic.go:286] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0504 20:46:41.889771 1 diskrecorder.go:223] Found files to send: insights-2026-05-04-204533.tar.gz I0504 20:46:41.889820 1 insightsuploader.go:150] Checking archives to upload periodically every 15m40.284668692s I0504 20:46:41.889828 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0504 20:46:41.900514 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0504 20:46:42.113473 1 requests.go:87] Successfully reported id=2026-05-04T20:46:41Z x-rh-insights-request-id=2f676ad475b84cc9ba15296832e2d516, wrote=52441 I0504 20:46:42.113631 1 insightsuploader.go:187] Uploaded report successfully in 223.795843ms I0504 20:46:42.113661 1 controller.go:129] Initializing last reported time to 2026-05-04T20:46:41Z I0504 20:46:42.113681 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0504 20:46:42.113689 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0504 20:46:42.113696 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0504 20:46:42.119837 1 controller.go:554] The operator is healthy I0504 20:47:11.931684 1 controller.go:554] The operator is healthy I0504 20:47:44.169833 1 insightsreport.go:137] Pulling report from smart-proxy I0504 20:47:44.169861 1 insightsreport.go:149] Retrieving report I0504 20:47:44.177208 1 requests.go:110] Retrieving report for cluster: 2ff01467-642b-47a6-8f11-228bcaaf9610 I0504 20:47:44.177221 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/2ff01467-642b-47a6-8f11-228bcaaf9610/reports I0504 20:47:44.180484 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/2ff01467-642b-47a6-8f11-228bcaaf9610/reports I0504 20:47:44.409624 1 insightsreport.go:184] Report retrieved I0504 20:47:44.438813 1 insightsreport.go:239] Report retrieved correctly I0504 20:49:11.930224 1 controller.go:554] The operator is healthy I0504 20:50:11.879950 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0504 20:50:11.888861 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 20:50:11.888881 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0504 20:50:11.893200 1 secretconfigobserver.go:119] support secret does not exist I0504 20:51:11.929581 1 controller.go:554] The operator is healthy I0504 20:53:11.929670 1 controller.go:554] The operator is healthy I0504 20:55:11.875420 1 configmapobserver.go:84] configmaps "insights-config" not found I0504 20:55:11.879925 1 configmapobserver.go:84] configmaps "insights-config" not found I0504 20:55:11.894015 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0504 20:55:11.900884 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 20:55:11.900900 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0504 20:55:11.909405 1 secretconfigobserver.go:119] support secret does not exist I0504 20:55:11.932339 1 controller.go:554] The operator is healthy I0504 20:57:11.930281 1 controller.go:554] The operator is healthy I0504 20:59:11.929703 1 controller.go:554] The operator is healthy I0504 21:00:11.910431 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0504 21:00:11.916246 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0504 21:00:11.916264 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0504 21:00:11.920476 1 secretconfigobserver.go:119] support secret does not exist I0504 21:01:11.931460 1 controller.go:554] The operator is healthy I0504 21:02:27.399361 1 insightsuploader.go:146] Nothing to report since 2026-05-04T20:46:41Z