I0327 23:01:24.193799 1 cmd.go:253] Using service-serving-cert provided certificates I0327 23:01:24.194295 1 observer_polling.go:159] Starting file observer I0327 23:01:24.751363 1 operator.go:60] Starting insights-operator v0.0.0-master+$Format:%H$ I0327 23:01:24.752506 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0327 23:01:24.752973 1 secure_serving.go:57] Forcing use of http/1.1 only W0327 23:01:24.752993 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0327 23:01:24.752998 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0327 23:01:24.753002 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0327 23:01:24.753005 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0327 23:01:24.753007 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0327 23:01:24.753010 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0327 23:01:24.753167 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0327 23:01:24.757001 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"99a57790-bd91-4634-90f9-43e03484d271", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}} I0327 23:01:24.756986 1 operator.go:125] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GatewayAPI GatewayAPIController HighlyAvailableArbiter ImageVolume IngressControllerLBSubnetsAWS KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController SigstoreImageVerification StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPClusterHostedDNSInstall GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall ImageModeStatusReporting ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesAzure ManagedBootImagesvSphere MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutatingAdmissionPolicy NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterOperations NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PreconfiguredUDNAddresses SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeAttributesClass VolumeGroupSnapshot] I0327 23:01:24.757050 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0327 23:01:24.757069 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0327 23:01:24.757065 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0327 23:01:24.757074 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0327 23:01:24.757082 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0327 23:01:24.757087 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0327 23:01:24.757289 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0327 23:01:24.757674 1 secure_serving.go:213] Serving securely on [::]:8443 I0327 23:01:24.757709 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0327 23:01:24.764177 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0327 23:01:24.764208 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0327 23:01:24.764313 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0327 23:01:24.768031 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0327 23:01:24.768047 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0327 23:01:24.771413 1 secretconfigobserver.go:119] support secret does not exist I0327 23:01:24.775150 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0327 23:01:24.778228 1 secretconfigobserver.go:119] support secret does not exist I0327 23:01:24.780840 1 recorder.go:161] Pruning old reports every 5h26m23s, max age is 288h0m0s I0327 23:01:24.784497 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0327 23:01:24.784507 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0327 23:01:24.784512 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0327 23:01:24.784516 1 insightsreport.go:296] Starting report retriever I0327 23:01:24.784527 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0327 23:01:24.784538 1 periodic.go:209] Running clusterconfig gatherer I0327 23:01:24.784580 1 tasks_processing.go:45] number of workers: 64 I0327 23:01:24.784608 1 tasks_processing.go:69] worker 4 listening for tasks. I0327 23:01:24.784621 1 tasks_processing.go:69] worker 2 listening for tasks. I0327 23:01:24.784620 1 tasks_processing.go:69] worker 1 listening for tasks. I0327 23:01:24.784629 1 tasks_processing.go:69] worker 0 listening for tasks. I0327 23:01:24.784629 1 tasks_processing.go:69] worker 3 listening for tasks. I0327 23:01:24.784636 1 tasks_processing.go:69] worker 23 listening for tasks. I0327 23:01:24.784636 1 tasks_processing.go:69] worker 32 listening for tasks. I0327 23:01:24.784640 1 tasks_processing.go:69] worker 24 listening for tasks. I0327 23:01:24.784645 1 tasks_processing.go:69] worker 5 listening for tasks. I0327 23:01:24.784640 1 tasks_processing.go:69] worker 8 listening for tasks. I0327 23:01:24.784649 1 tasks_processing.go:69] worker 14 listening for tasks. I0327 23:01:24.784648 1 tasks_processing.go:69] worker 13 listening for tasks. I0327 23:01:24.784656 1 tasks_processing.go:69] worker 15 listening for tasks. I0327 23:01:24.784656 1 tasks_processing.go:69] worker 25 listening for tasks. I0327 23:01:24.784661 1 tasks_processing.go:69] worker 26 listening for tasks. I0327 23:01:24.784662 1 tasks_processing.go:69] worker 9 listening for tasks. I0327 23:01:24.784658 1 tasks_processing.go:69] worker 6 listening for tasks. I0327 23:01:24.784666 1 tasks_processing.go:69] worker 16 listening for tasks. I0327 23:01:24.784668 1 tasks_processing.go:69] worker 10 listening for tasks. I0327 23:01:24.784671 1 tasks_processing.go:69] worker 17 listening for tasks. I0327 23:01:24.784672 1 tasks_processing.go:69] worker 11 listening for tasks. I0327 23:01:24.784675 1 tasks_processing.go:69] worker 12 listening for tasks. I0327 23:01:24.784678 1 tasks_processing.go:69] worker 21 listening for tasks. I0327 23:01:24.784682 1 tasks_processing.go:69] worker 18 listening for tasks. I0327 23:01:24.784685 1 tasks_processing.go:69] worker 22 listening for tasks. I0327 23:01:24.784688 1 tasks_processing.go:69] worker 19 listening for tasks. I0327 23:01:24.784689 1 tasks_processing.go:69] worker 29 listening for tasks. I0327 23:01:24.784694 1 tasks_processing.go:69] worker 31 listening for tasks. I0327 23:01:24.784692 1 tasks_processing.go:69] worker 7 listening for tasks. I0327 23:01:24.784694 1 tasks_processing.go:69] worker 20 listening for tasks. I0327 23:01:24.784703 1 tasks_processing.go:69] worker 33 listening for tasks. I0327 23:01:24.784701 1 tasks_processing.go:69] worker 27 listening for tasks. I0327 23:01:24.784709 1 tasks_processing.go:71] worker 23 working on tsdb_status task. I0327 23:01:24.784706 1 tasks_processing.go:69] worker 34 listening for tasks. I0327 23:01:24.784711 1 tasks_processing.go:69] worker 30 listening for tasks. I0327 23:01:24.784719 1 tasks_processing.go:71] worker 34 working on openstack_dataplanedeployments task. I0327 23:01:24.784716 1 tasks_processing.go:69] worker 47 listening for tasks. I0327 23:01:24.784699 1 tasks_processing.go:69] worker 35 listening for tasks. I0327 23:01:24.784727 1 tasks_processing.go:71] worker 21 working on operators task. I0327 23:01:24.784727 1 tasks_processing.go:69] worker 39 listening for tasks. I0327 23:01:24.784730 1 tasks_processing.go:71] worker 22 working on sap_config task. I0327 23:01:24.784726 1 tasks_processing.go:71] worker 15 working on aggregated_monitoring_cr_names task. I0327 23:01:24.784740 1 tasks_processing.go:71] worker 7 working on networks task. I0327 23:01:24.784742 1 tasks_processing.go:69] worker 56 listening for tasks. I0327 23:01:24.784742 1 tasks_processing.go:71] worker 2 working on crds task. I0327 23:01:24.784743 1 tasks_processing.go:69] worker 46 listening for tasks. I0327 23:01:24.784750 1 tasks_processing.go:69] worker 57 listening for tasks. I0327 23:01:24.784745 1 tasks_processing.go:69] worker 50 listening for tasks. I0327 23:01:24.784757 1 tasks_processing.go:71] worker 10 working on sap_datahubs task. I0327 23:01:24.784759 1 tasks_processing.go:69] worker 58 listening for tasks. I0327 23:01:24.784763 1 tasks_processing.go:71] worker 20 working on nodenetworkconfigurationpolicies task. I0327 23:01:24.784767 1 tasks_processing.go:69] worker 59 listening for tasks. I0327 23:01:24.784775 1 tasks_processing.go:69] worker 60 listening for tasks. I0327 23:01:24.784769 1 tasks_processing.go:69] worker 41 listening for tasks. I0327 23:01:24.784717 1 tasks_processing.go:69] worker 44 listening for tasks. I0327 23:01:24.784784 1 tasks_processing.go:69] worker 61 listening for tasks. I0327 23:01:24.784788 1 tasks_processing.go:71] worker 41 working on schedulers task. I0327 23:01:24.784790 1 tasks_processing.go:71] worker 17 working on container_runtime_configs task. I0327 23:01:24.784792 1 tasks_processing.go:69] worker 40 listening for tasks. I0327 23:01:24.784794 1 tasks_processing.go:71] worker 61 working on image task. I0327 23:01:24.784797 1 tasks_processing.go:71] worker 24 working on mutating_webhook_configurations task. I0327 23:01:24.784800 1 tasks_processing.go:71] worker 40 working on ingress_certificates task. I0327 23:01:24.784803 1 tasks_processing.go:69] worker 42 listening for tasks. I0327 23:01:24.784804 1 tasks_processing.go:71] worker 6 working on openstack_controlplanes task. I0327 23:01:24.784808 1 tasks_processing.go:71] worker 9 working on oauths task. I0327 23:01:24.784805 1 tasks_processing.go:71] worker 11 working on qemu_kubevirt_launcher_logs task. I0327 23:01:24.784808 1 tasks_processing.go:69] worker 62 listening for tasks. I0327 23:01:24.784820 1 tasks_processing.go:71] worker 16 working on overlapping_namespace_uids task. I0327 23:01:24.784821 1 tasks_processing.go:71] worker 42 working on openstack_version task. I0327 23:01:24.784827 1 tasks_processing.go:71] worker 62 working on cost_management_metrics_configs task. I0327 23:01:24.784826 1 tasks_processing.go:71] worker 13 working on certificate_signing_requests task. I0327 23:01:24.784828 1 tasks_processing.go:71] worker 30 working on dvo_metrics task. I0327 23:01:24.784837 1 tasks_processing.go:71] worker 59 working on pdbs task. I0327 23:01:24.784792 1 tasks_processing.go:71] worker 5 working on version task. I0327 23:01:24.784712 1 tasks_processing.go:69] worker 63 listening for tasks. I0327 23:01:24.784883 1 tasks_processing.go:69] worker 43 listening for tasks. I0327 23:01:24.784700 1 tasks_processing.go:69] worker 36 listening for tasks. I0327 23:01:24.784888 1 tasks_processing.go:71] worker 0 working on clusterroles task. I0327 23:01:24.784893 1 tasks_processing.go:71] worker 63 working on ceph_cluster task. I0327 23:01:24.784896 1 tasks_processing.go:71] worker 35 working on monitoring_persistent_volumes task. I0327 23:01:24.784894 1 tasks_processing.go:71] worker 43 working on machine_config_pools task. I0327 23:01:24.784905 1 tasks_processing.go:71] worker 47 working on image_registries task. I0327 23:01:24.785036 1 tasks_processing.go:71] worker 60 working on pod_network_connectivity_checks task. I0327 23:01:24.784734 1 tasks_processing.go:69] worker 55 listening for tasks. I0327 23:01:24.784717 1 tasks_processing.go:69] worker 52 listening for tasks. I0327 23:01:24.784720 1 tasks_processing.go:71] worker 25 working on node_logs task. I0327 23:01:24.785103 1 tasks_processing.go:71] worker 3 working on nodes task. I0327 23:01:24.785219 1 tasks_processing.go:71] worker 39 working on feature_gates task. I0327 23:01:24.785284 1 tasks_processing.go:71] worker 56 working on ingress task. I0327 23:01:24.784833 1 tasks_processing.go:71] worker 58 working on silenced_alerts task. I0327 23:01:24.784712 1 tasks_processing.go:69] worker 51 listening for tasks. I0327 23:01:24.784721 1 tasks_processing.go:69] worker 45 listening for tasks. I0327 23:01:24.784723 1 tasks_processing.go:71] worker 26 working on machines task. I0327 23:01:24.785487 1 tasks_processing.go:71] worker 45 working on storage_classes task. I0327 23:01:24.784722 1 tasks_processing.go:69] worker 53 listening for tasks. I0327 23:01:24.784726 1 tasks_processing.go:71] worker 18 working on metrics task. I0327 23:01:24.785524 1 tasks_processing.go:71] worker 53 working on service_accounts task. I0327 23:01:24.785119 1 tasks_processing.go:71] worker 52 working on machine_configs task. I0327 23:01:24.784721 1 tasks_processing.go:69] worker 48 listening for tasks. I0327 23:01:24.784733 1 tasks_processing.go:71] worker 1 working on config_maps task. I0327 23:01:24.784732 1 tasks_processing.go:71] worker 19 working on jaegers task. I0327 23:01:24.784727 1 tasks_processing.go:69] worker 54 listening for tasks. I0327 23:01:24.784737 1 tasks_processing.go:71] worker 31 working on storage_cluster task. I0327 23:01:24.784737 1 tasks_processing.go:71] worker 4 working on machine_autoscalers task. I0327 23:01:24.784694 1 tasks_processing.go:69] worker 28 listening for tasks. I0327 23:01:24.784776 1 tasks_processing.go:71] worker 33 working on openshift_logging task. I0327 23:01:24.784734 1 tasks_processing.go:69] worker 49 listening for tasks. I0327 23:01:24.784793 1 tasks_processing.go:71] worker 32 working on machine_healthchecks task. I0327 23:01:24.784735 1 tasks_processing.go:71] worker 29 working on validating_webhook_configurations task. I0327 23:01:24.784817 1 tasks_processing.go:71] worker 8 working on nodenetworkstates task. I0327 23:01:24.784822 1 tasks_processing.go:71] worker 14 working on olm_operators task. I0327 23:01:24.784819 1 tasks_processing.go:71] worker 46 working on image_pruners task. I0327 23:01:24.784825 1 tasks_processing.go:71] worker 57 working on openstack_dataplanenodesets task. I0327 23:01:24.784705 1 tasks_processing.go:69] worker 37 listening for tasks. I0327 23:01:24.784892 1 tasks_processing.go:71] worker 12 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0327 23:01:24.786179 1 tasks_processing.go:71] worker 37 working on container_images task. I0327 23:01:24.784829 1 tasks_processing.go:71] worker 50 working on sap_pods task. I0327 23:01:24.784716 1 tasks_processing.go:71] worker 27 working on openshift_machine_api_events task. I0327 23:01:24.785128 1 tasks_processing.go:71] worker 55 working on operators_pods_and_events task. I0327 23:01:24.784696 1 tasks_processing.go:69] worker 38 listening for tasks. I0327 23:01:24.786615 1 tasks_processing.go:71] worker 38 working on infrastructures task. I0327 23:01:24.785483 1 tasks_processing.go:71] worker 51 working on active_alerts task. I0327 23:01:24.784789 1 tasks_processing.go:71] worker 44 working on cluster_apiserver task. I0327 23:01:24.785967 1 tasks_processing.go:71] worker 54 working on lokistack task. I0327 23:01:24.785973 1 tasks_processing.go:71] worker 48 working on machine_sets task. I0327 23:01:24.785974 1 tasks_processing.go:71] worker 49 working on proxies task. I0327 23:01:24.785979 1 tasks_processing.go:71] worker 28 working on support_secret task. I0327 23:01:24.785124 1 tasks_processing.go:71] worker 36 working on authentication task. I0327 23:01:24.787529 1 tasks_processing.go:71] worker 6 working on install_plans task. I0327 23:01:24.787554 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 2.711375ms to process 0 records I0327 23:01:24.788113 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 3.292522ms to process 0 records I0327 23:01:24.788170 1 tasks_processing.go:74] worker 20 stopped. I0327 23:01:24.788626 1 tasks_processing.go:74] worker 63 stopped. I0327 23:01:24.788627 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 3.702338ms to process 0 records I0327 23:01:24.788976 1 tasks_processing.go:74] worker 42 stopped. I0327 23:01:24.788992 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 4.141527ms to process 0 records E0327 23:01:24.790280 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.63.68:9091: connect: connection refused I0327 23:01:24.790299 1 tasks_processing.go:74] worker 23 stopped. E0327 23:01:24.790305 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.63.68:9091: connect: connection refused I0327 23:01:24.790314 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 5.580909ms to process 0 records E0327 23:01:24.790311 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.63.68:9091: connect: connection refused I0327 23:01:24.790333 1 tasks_processing.go:74] worker 18 stopped. E0327 23:01:24.790341 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.63.68:9091: connect: connection refused I0327 23:01:24.790356 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 4.797916ms to process 0 records I0327 23:01:24.793649 1 tasks_processing.go:74] worker 17 stopped. I0327 23:01:24.793661 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 8.85026ms to process 0 records I0327 23:01:24.793670 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 8.933655ms to process 0 records I0327 23:01:24.793679 1 tasks_processing.go:74] worker 34 stopped. I0327 23:01:24.793727 1 tasks_processing.go:74] worker 43 stopped. I0327 23:01:24.793742 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 8.819148ms to process 0 records E0327 23:01:24.793758 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0327 23:01:24.793767 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 8.683148ms to process 0 records I0327 23:01:24.793780 1 tasks_processing.go:74] worker 60 stopped. I0327 23:01:24.793927 1 controller.go:128] Initializing last reported time to 0001-01-01T00:00:00Z I0327 23:01:24.793969 1 controller.go:326] The initial operator extension status is healthy I0327 23:01:24.793981 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0327 23:01:24.793985 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0327 23:01:24.793988 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0327 23:01:24.794003 1 controller.go:489] The operator is still being initialized I0327 23:01:24.794008 1 controller.go:512] The operator is healthy I0327 23:01:24.794112 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0327 23:01:24.793969 1 tasks_processing.go:74] worker 59 stopped. I0327 23:01:24.794121 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s I0327 23:01:24.794165 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=474eedf604f63dd31cc51614959d88f4a700e7d2abf1b2eeebb7f44dd2b6a32b I0327 23:01:24.794195 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=b7d2a85005c5f1c25bdce15f7dcea7bad5c17ce54b76a438e5efebba80a953fa W0327 23:01:24.794216 1 operator.go:288] started I0327 23:01:24.794221 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=8c8d064dda9efc72883874f5720464ab26f816605e494c2e71bb33270d6d794c I0327 23:01:24.794237 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0327 23:01:24.794245 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=abf94c39e249d001749f429d617c055cfc2815f478ac5c75a8ce235ce02e60f7 I0327 23:01:24.794270 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=59b4d0f2ff338655c71580905101727baaa1e971cb32c4af253e67c9f31fca86 I0327 23:01:24.794293 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=ab0fcee5292aef47215335a29ac21d2a0aef5ca7b53fbeb66f5fa892631e2d6e I0327 23:01:24.794317 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=60f615d5b8c22de721e6e5f1751b6e3ca8c82777765c14ee5ec2fc23743bdba1 I0327 23:01:24.794343 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=1b39ffaeec2c182186b0cb493ff9679b452268f2a70bad7a096a44183976c1c4 I0327 23:01:24.794369 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=091f4c8689db18fe1f1a3df69bf844e614b1fb14964112fb4753f043fd6af04c I0327 23:01:24.794400 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=b70f7bfca7aded0627cb7d4dda93166c7878c9319154cacb74fe5628dca7fedb I0327 23:01:24.794426 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=2b59fe0bfac50d2ef5e5c388ee9153a9c706458ba66744451359c55e2c433c79 I0327 23:01:24.794437 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 9.091563ms to process 11 records I0327 23:01:24.794525 1 tasks_processing.go:74] worker 9 stopped. I0327 23:01:24.794751 1 recorder.go:75] Recording config/oauth with fingerprint=8a2592900799d0b82c84d093a46e7c1b8b103bfb2c679282dca6d41fde83be7f I0327 23:01:24.794766 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 9.176931ms to process 1 records I0327 23:01:24.801176 1 tasks_processing.go:74] worker 22 stopped. I0327 23:01:24.801188 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 16.431243ms to process 0 records I0327 23:01:24.806810 1 tasks_processing.go:74] worker 47 stopped. I0327 23:01:24.807226 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=3707defadfbf88de28ff6fe8cb71917ad3bdc102ef1d194b2a8c0473a6ed3ff6 I0327 23:01:24.807240 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 21.890636ms to process 1 records I0327 23:01:24.811765 1 tasks_processing.go:74] worker 33 stopped. I0327 23:01:24.811788 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 25.984084ms to process 0 records I0327 23:01:24.811799 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 26.936609ms to process 0 records I0327 23:01:24.811808 1 tasks_processing.go:74] worker 62 stopped. I0327 23:01:24.811894 1 tasks_processing.go:74] worker 26 stopped. E0327 23:01:24.811908 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0327 23:01:24.811916 1 gather.go:177] gatherer "clusterconfig" function "machines" took 26.401245ms to process 0 records I0327 23:01:24.812008 1 tasks_processing.go:74] worker 56 stopped. I0327 23:01:24.812124 1 recorder.go:75] Recording config/ingress with fingerprint=f690ff86b49bc1f4eeea5262f304423e992fe59fccf898dbb3758056a83c2c0a I0327 23:01:24.812135 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 26.670181ms to process 1 records I0327 23:01:24.812222 1 tasks_processing.go:74] worker 7 stopped. I0327 23:01:24.812268 1 recorder.go:75] Recording config/network with fingerprint=706f03eb9847dcd2329e820f3be231e98be00931000ca0c06292f8e33dc2f33e I0327 23:01:24.812288 1 gather.go:177] gatherer "clusterconfig" function "networks" took 27.273154ms to process 1 records I0327 23:01:24.812534 1 tasks_processing.go:74] worker 35 stopped. I0327 23:01:24.812547 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 27.629619ms to process 0 records I0327 23:01:24.812564 1 tasks_processing.go:74] worker 8 stopped. I0327 23:01:24.812581 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 26.521217ms to process 0 records I0327 23:01:24.812601 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 27.50443ms to process 0 records I0327 23:01:24.812610 1 tasks_processing.go:74] worker 25 stopped. I0327 23:01:24.812713 1 tasks_processing.go:74] worker 39 stopped. I0327 23:01:24.812807 1 recorder.go:75] Recording config/featuregate with fingerprint=576c62cbe112aa653b9b4a887707e749f04f34650f8ca03a190fcd968032cb42 I0327 23:01:24.812818 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 27.43794ms to process 1 records I0327 23:01:24.813240 1 tasks_processing.go:74] worker 19 stopped. I0327 23:01:24.813251 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 27.56201ms to process 0 records I0327 23:01:24.813423 1 tasks_processing.go:74] worker 45 stopped. I0327 23:01:24.813524 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=d08ad419c4e8210be796be269dd4c6c696ce22d2453d114c17d001f4e132f9b1 I0327 23:01:24.813546 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=eba56e360f9b95f52fda936e0eae24369478418aea38c49b99e7ab3f4b2cd8a6 I0327 23:01:24.813556 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 27.921272ms to process 2 records I0327 23:01:24.813589 1 tasks_processing.go:74] worker 44 stopped. I0327 23:01:24.813700 1 recorder.go:75] Recording config/apiserver with fingerprint=6490922e5d8dec18412c7f7edc629645f646e733f066dc8d41e7a113dc20cb15 I0327 23:01:24.813711 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 26.925144ms to process 1 records I0327 23:01:24.813785 1 recorder.go:75] Recording config/image with fingerprint=2a3d12d1675a98bd8d4543c4cdd3d0eb9e89ca0e73dca510af10c4acb99a2977 I0327 23:01:24.813795 1 gather.go:177] gatherer "clusterconfig" function "image" took 28.798276ms to process 1 records I0327 23:01:24.813800 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 27.334891ms to process 0 records I0327 23:01:24.813831 1 tasks_processing.go:74] worker 61 stopped. I0327 23:01:24.813841 1 tasks_processing.go:74] worker 50 stopped. I0327 23:01:24.813869 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=884242ddb1947224cccf5cc4a0769f3e76fd7046391dfd25a87c54a0c41734c8 I0327 23:01:24.813878 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 27.60142ms to process 1 records I0327 23:01:24.813882 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 27.40781ms to process 0 records I0327 23:01:24.813899 1 tasks_processing.go:74] worker 46 stopped. I0327 23:01:24.813907 1 tasks_processing.go:74] worker 27 stopped. I0327 23:01:24.813937 1 recorder.go:75] Recording config/proxy with fingerprint=35527dc4f420be24eb9f4d821ec316e58e5e8f2330b1f8d7f4b37d36694d96fe I0327 23:01:24.813958 1 tasks_processing.go:74] worker 49 stopped. I0327 23:01:24.813966 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 26.750859ms to process 1 records I0327 23:01:24.814040 1 tasks_processing.go:74] worker 38 stopped. W0327 23:01:24.814333 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0327 23:01:24.814396 1 recorder.go:75] Recording config/infrastructure with fingerprint=1125f8fce70ff9bd8f966c4750af4bea39a288b6e16405b5563aa3232fbaa8c2 I0327 23:01:24.814404 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 27.217032ms to process 1 records I0327 23:01:24.815087 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 29.362337ms to process 0 records I0327 23:01:24.815097 1 tasks_processing.go:74] worker 31 stopped. I0327 23:01:24.815251 1 tasks_processing.go:74] worker 48 stopped. I0327 23:01:24.815273 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 28.44935ms to process 0 records I0327 23:01:24.815293 1 tasks_processing.go:74] worker 4 stopped. I0327 23:01:24.815317 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 29.561793ms to process 0 records I0327 23:01:24.815628 1 tasks_processing.go:74] worker 41 stopped. I0327 23:01:24.815744 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=7f26be204755ce214f28da6e2d1062f1a3591aa9c7169aadded1a9a0a3b4eedb I0327 23:01:24.815764 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 30.830972ms to process 1 records I0327 23:01:24.815811 1 gather_logs.go:145] no pods in namespace were found I0327 23:01:24.815821 1 tasks_processing.go:74] worker 11 stopped. I0327 23:01:24.815829 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 30.998811ms to process 0 records I0327 23:01:24.815844 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 29.081057ms to process 0 records I0327 23:01:24.815850 1 tasks_processing.go:74] worker 54 stopped. I0327 23:01:24.815908 1 tasks_processing.go:74] worker 32 stopped. E0327 23:01:24.815914 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0327 23:01:24.815931 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 30.043307ms to process 0 records E0327 23:01:24.815977 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0327 23:01:24.815985 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 28.61013ms to process 0 records I0327 23:01:24.815992 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 29.843445ms to process 0 records I0327 23:01:24.815999 1 tasks_processing.go:74] worker 57 stopped. I0327 23:01:24.816005 1 tasks_processing.go:74] worker 28 stopped. I0327 23:01:24.816017 1 tasks_processing.go:74] worker 24 stopped. I0327 23:01:24.816568 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=d02e489206394969d42efee84a5f0961761c92e60af81ec91021381d77d11ed3 I0327 23:01:24.816612 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=5c215b4b8c087a99c447201401b649f4278fadddd3e703625013e77aa3e7ddd8 I0327 23:01:24.816655 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=7499c3c9bf3eb679b0affad86881d937da256a6836045fb7050d0d14e171d501 I0327 23:01:24.816667 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 31.208308ms to process 3 records I0327 23:01:24.816768 1 tasks_processing.go:74] worker 36 stopped. I0327 23:01:24.817103 1 recorder.go:75] Recording config/authentication with fingerprint=2077659ae967e0a5d0ec75ea9c55b19505760492b8105bae83f74b32cb7c6a86 I0327 23:01:24.817143 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 28.939232ms to process 1 records I0327 23:01:24.817163 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 31.715156ms to process 0 records I0327 23:01:24.817177 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 31.053834ms to process 0 records I0327 23:01:24.817198 1 tasks_processing.go:74] worker 14 stopped. I0327 23:01:24.817220 1 tasks_processing.go:74] worker 10 stopped. I0327 23:01:24.817242 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0327 23:01:24.817264 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0327 23:01:24.817269 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0327 23:01:24.817274 1 controller.go:212] Source scaController *sca.Controller is not ready I0327 23:01:24.817279 1 controller.go:212] Source clusterTransferController *clustertransfer.Controller is not ready I0327 23:01:24.817329 1 controller.go:489] The operator is still being initialized I0327 23:01:24.817343 1 controller.go:512] The operator is healthy I0327 23:01:24.818044 1 tasks_processing.go:74] worker 3 stopped. I0327 23:01:24.819273 1 recorder.go:75] Recording config/node/ip-10-0-0-195.ec2.internal with fingerprint=515dde8387a4c5215cdeb606b0e686e1416663f1189a5c10f9326de68f11e877 I0327 23:01:24.819390 1 recorder.go:75] Recording config/node/ip-10-0-1-9.ec2.internal with fingerprint=5762319e26b9d3d6891dde89063e63bcae0b1af757ca24802341ba8ec6ab7aca I0327 23:01:24.819488 1 recorder.go:75] Recording config/node/ip-10-0-2-6.ec2.internal with fingerprint=990883b51a7b7b41886afa8317c2b07106aeead5af1c4c34a19b713ab7ce03c1 I0327 23:01:24.819502 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 32.882169ms to process 3 records I0327 23:01:24.819521 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0327 23:01:24.819530 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 34.037ms to process 1 records I0327 23:01:24.819538 1 tasks_processing.go:74] worker 16 stopped. I0327 23:01:24.820117 1 tasks_processing.go:74] worker 15 stopped. I0327 23:01:24.820132 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 35.369165ms to process 0 records I0327 23:01:24.820144 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 35.290685ms to process 0 records I0327 23:01:24.820150 1 tasks_processing.go:74] worker 13 stopped. I0327 23:01:24.820258 1 tasks_processing.go:74] worker 29 stopped. I0327 23:01:24.820428 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=3aa17ae64c5551824e93c927308e5ac92d306a2b36e8044ea2dd2d69c8ea4ce5 I0327 23:01:24.820497 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=db19041b788f6e79f15f981ba97e666801eba5ab44c812aa3a2e2253f488eaea I0327 23:01:24.820606 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=555ea6b5461ebb48c17a4c24e2fc067ee9e499f197d3196bb990755be6174b78 I0327 23:01:24.820642 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=4f2c8caed82e0776883ac45fc9d757e3dbb0a57c46c78a1a41acc787c4732b5c I0327 23:01:24.820674 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=81976ba77489732434ee9ac416906ab555121b8df866357b732e5328d8a48e0c I0327 23:01:24.820712 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=a9b3fbcc27e05cd103b2268abbb8c33483891dd8e9fe40f32fe1214036ba5f03 I0327 23:01:24.820741 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=555562e147e6ac4130a98fa2d8d34a910d0803cb650e854bbc402d72182c990a I0327 23:01:24.820786 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=769bc0a02a866f4a27e192312efc031277354b5fc4daaee4a04d192ea0083051 I0327 23:01:24.820814 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=a86ae24383a77a054d0502a820c756afc06493d121769ead5a82f8ab1c38cb5f I0327 23:01:24.820871 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=264a47357d7f56eea78369a8db944c73aef45803cc0ba968d36f4bb38f90e06c I0327 23:01:24.820905 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=224d46f17f4448ab5a91b53a3caec019e9026e076638d68c92ae905ec64c17df I0327 23:01:24.820974 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=8121fad4bec2ab23e3415f3c5422cdaf3729573096da363bb2e21f868df4a6ff I0327 23:01:24.821012 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=52e3b6b01254cdac9f43f3c20f70cd724454ba0ddf561a73ddd81574e92a4ede I0327 23:01:24.821024 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 34.24868ms to process 13 records I0327 23:01:24.823182 1 tasks_processing.go:74] worker 0 stopped. I0327 23:01:24.823445 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=05b67f3953b6bf412ed28278c9dbfb70c41b2f1ba537db062ffa6f0391072d61 I0327 23:01:24.823561 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=7ac9d2630ca3e93d1e40ac4b4eb3d570bddc507129b7ef06f5da95d0b06ee44e I0327 23:01:24.823579 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 38.272697ms to process 2 records I0327 23:01:24.825514 1 tasks_processing.go:74] worker 5 stopped. I0327 23:01:24.826743 1 recorder.go:75] Recording config/version with fingerprint=8c6b117a5ecd739f462fa05994f9ab38d35e4f4f91e49a7271863421fc06dc50 I0327 23:01:24.826760 1 recorder.go:75] Recording config/id with fingerprint=2b337db9d896595364c233c0b7252b604208597f80c01fdfca8ffc091635824b I0327 23:01:24.826767 1 gather.go:177] gatherer "clusterconfig" function "version" took 40.657188ms to process 2 records I0327 23:01:24.826902 1 tasks_processing.go:74] worker 2 stopped. I0327 23:01:24.827298 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=23e6bbf50aef8a3bd7c872cf946bbe0b13fc75a6bb3af58d22746df8a16c6bc0 I0327 23:01:24.827525 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=af4fb972bc77bfbb9a7548c1abce55175fbddfb4ebe35caa45769f8b7c0f488c I0327 23:01:24.827536 1 gather.go:177] gatherer "clusterconfig" function "crds" took 40.888528ms to process 2 records I0327 23:01:24.827544 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 40.390706ms to process 0 records I0327 23:01:24.827550 1 tasks_processing.go:74] worker 55 stopped. I0327 23:01:24.829431 1 tasks_processing.go:74] worker 52 stopped. I0327 23:01:24.829459 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0327 23:01:24.829467 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 43.86748ms to process 1 records I0327 23:01:24.835903 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0327 23:01:24.837039 1 tasks_processing.go:74] worker 40 stopped. E0327 23:01:24.837054 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0327 23:01:24.837068 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2pabjllrb2mlslikfub2i3bn689ace52-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2pabjllrb2mlslikfub2i3bn689ace52-primary-cert-bundle-secret" not found I0327 23:01:24.837185 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=7ada5cbe4e3148b8555533d50d5d633fed8aa9d64496dcb0e77b5951b27e5a23 I0327 23:01:24.837205 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 52.221214ms to process 1 records I0327 23:01:24.838712 1 tasks_processing.go:74] worker 1 stopped. E0327 23:01:24.838779 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0327 23:01:24.838815 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0327 23:01:24.838823 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0327 23:01:24.838846 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=26b6661162b099a0f5a279859b4f46c867929a79d9a4a41fde4be4e6fe138018 I0327 23:01:24.838915 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0327 23:01:24.838930 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0327 23:01:24.838968 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=c93090eb0d2a4736885abeb79c91680cfd01fda46464f83456b085d4dc8239f0 I0327 23:01:24.839045 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0327 23:01:24.839106 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0327 23:01:24.839119 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0327 23:01:24.839127 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 53.094954ms to process 7 records I0327 23:01:24.840330 1 tasks_processing.go:74] worker 12 stopped. I0327 23:01:24.840345 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 54.143689ms to process 0 records I0327 23:01:24.841697 1 tasks_processing.go:74] worker 37 stopped. I0327 23:01:24.843320 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-575cd97545-d47wz with fingerprint=72197b1acc3e28c7d2e4ed154d294f4ddbe20a693679edca5e03bf27f4f6a760 I0327 23:01:24.843447 1 recorder.go:75] Recording config/running_containers with fingerprint=485ec8c3c8708ab6d6eab366d6db5337a8cdbcd091f8c13aa7cd4b215f4e7e4f I0327 23:01:24.843506 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 55.499458ms to process 2 records I0327 23:01:24.857770 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0327 23:01:24.857776 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0327 23:01:24.857795 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0327 23:01:24.864896 1 base_controller.go:82] Caches are synced for ConfigController I0327 23:01:24.864907 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0327 23:01:24.868421 1 configmapobserver.go:84] configmaps "insights-config" not found I0327 23:01:24.894704 1 base_controller.go:82] Caches are synced for LoggingSyncer I0327 23:01:24.894717 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0327 23:01:24.915825 1 cluster_transfer.go:102] no available accepted cluster transfer I0327 23:01:24.915840 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0327 23:01:25.012060 1 sca.go:266] etc-pki-entitlement secret successfully created I0327 23:01:25.012080 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated W0327 23:01:25.814873 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0327 23:01:25.869126 1 tasks_processing.go:74] worker 58 stopped. I0327 23:01:25.869145 1 recorder.go:75] Recording config/silenced_alerts.json with fingerprint=37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570 I0327 23:01:25.869158 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 1.083706754s to process 1 records I0327 23:01:25.869265 1 tasks_processing.go:74] worker 51 stopped. I0327 23:01:25.869299 1 recorder.go:75] Recording config/alerts with fingerprint=63f257fdc6793ff77dc130a54d834e3e2101843915641b163fba7df14229ac02 I0327 23:01:25.869312 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 1.082628954s to process 1 records W0327 23:01:26.814079 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0327 23:01:27.243985 1 tasks_processing.go:74] worker 21 stopped. I0327 23:01:27.244124 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=649b49ff355f4a948e63664b5d191db6ca0be5a650b632a20251a1dcc9efeea2 I0327 23:01:27.244183 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0327 23:01:27.244222 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=7b84b523c6de31a369f9d59f070e843d2e9f7f6b872b68df3689cbcc05eb92c8 I0327 23:01:27.244238 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0327 23:01:27.244271 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=d87af849a08691db6738d3e4905f707bafa2b436c55ea5ea43e1687cdeb5b206 I0327 23:01:27.244297 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0327 23:01:27.244331 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=ae3a305a8cddbf4f8325fb7bffc0f1b44cd448198e3f5271014c333128cf0492 I0327 23:01:27.244370 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=e82528127c7f3920f9cad15945ef42490a1e329b54786bd430ac6ad0645f1312 I0327 23:01:27.244404 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=3e645d4ec7a32172a501b7f7aeee12a6545602f953bc57b8da7d0a9714168715 I0327 23:01:27.244446 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=f943bcd1f40d86cbbed4c93349270e53ccba390f378158da7481f4325acae9c8 I0327 23:01:27.244711 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=1d0709ebdf5dd3fb93ecba178c40d61d9a4cc51087cce557abd240f4026269a7 I0327 23:01:27.244743 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=90f8c5cd8e5963b4f59a4037f8f427d5f2fafb9e1871164bed76842995889f03 I0327 23:01:27.244757 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0327 23:01:27.244782 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=b5880ad1d978f65fd8be252264d0bab039f25d68e5065ea33115a2af7eec0093 I0327 23:01:27.244799 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0327 23:01:27.244824 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=a588eadba3f571f43e643415d12cdcc3ed2fce8668f761d00e05cb8eddadda1f I0327 23:01:27.244837 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0327 23:01:27.244870 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=1393f2feffbf9298c83a15ec7e27f9991510e226e45ab6793ac5396d5e2632e5 I0327 23:01:27.244884 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0327 23:01:27.244912 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=91a2d9bd57926391f43caa70fbb9d155d73ae53be614e50b80bbd094d98f8854 I0327 23:01:27.245045 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=20465aeb7f279237a497e87834a3f74b02b6cfe0d1ed69f4b1ad500772dc41e5 I0327 23:01:27.245061 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0327 23:01:27.245074 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0327 23:01:27.245105 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0327 23:01:27.245138 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=29da75a0b349353f1afdd889a64aa3cbc99f0dcfd6fef72411f4805986cc6f89 I0327 23:01:27.245175 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=aa50a00a2841da607bd39a488911fa4ccade22b0fe46ec198c3e2c8006b6ef68 I0327 23:01:27.245189 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0327 23:01:27.245213 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=019be9e1ac0f7a4f79b93a41ed00e8802ac23e62b18c20786853eda981a06bd7 I0327 23:01:27.245227 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0327 23:01:27.245252 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=defd42b7f94d513ed7412307383df90b5ce6ee48632977a9891edb8e377dc59d I0327 23:01:27.245266 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0327 23:01:27.245287 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=17e2469aa01ea3f9abdb730fc41f3f429891ce617469c344a3beffb900f4ba64 I0327 23:01:27.245312 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=650feb08d73620bad4fae1144d80ef2521e371dd8f7d9b9ecf489225c57ab03e I0327 23:01:27.245338 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=82e8601ff705a3256ed50430276837a5d450ec4f5d5b832b7bb0a41d90d140f6 I0327 23:01:27.245370 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=9047ec340c2279ced50f6a940a907e25b61c8d3fa1a63cb523a8546014e57ccd I0327 23:01:27.245385 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0327 23:01:27.245428 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=61759fe03681f2f0d9e8cd47acc5a4c1f28c697609c13ecab863a45d70bacc6a I0327 23:01:27.245454 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0327 23:01:27.245468 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0327 23:01:27.245480 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.459242391s to process 39 records W0327 23:01:27.814870 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0327 23:01:28.814306 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0327 23:01:29.814385 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0327 23:01:29.814412 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0327 23:01:29.814427 1 tasks_processing.go:74] worker 30 stopped. E0327 23:01:29.814438 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0327 23:01:29.814448 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0327 23:01:29.814459 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.029583044s to process 1 records I0327 23:01:37.625502 1 tasks_processing.go:74] worker 6 stopped. I0327 23:01:37.625542 1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b I0327 23:01:37.625555 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.837912253s to process 1 records I0327 23:01:38.391629 1 tasks_processing.go:74] worker 53 stopped. I0327 23:01:38.391899 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=230dfa4a1c30b0cc2ffe8f912592f10befbcd8eab365a68f86f7a7783043a01b I0327 23:01:38.391916 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.606087466s to process 1 records E0327 23:01:38.391997 1 periodic.go:247] "Unhandled Error" err="clusterconfig failed after 13.607s with: function \"tsdb_status\" failed with an error, function \"metrics\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machines\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"support_secret\" failed with an error, function \"ingress_certificates\" failed with an error, function \"config_maps\" failed with an error, function \"dvo_metrics\" failed with an error" I0327 23:01:38.393111 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "tsdb_status" failed with an error, function "metrics" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machines" failed with an error, function "machine_healthchecks" failed with an error, function "support_secret" failed with an error, function "ingress_certificates" failed with an error, function "config_maps" failed with an error, function "dvo_metrics" failed with an error I0327 23:01:38.393127 1 periodic.go:209] Running workloads gatherer I0327 23:01:38.393143 1 tasks_processing.go:45] number of workers: 2 I0327 23:01:38.393153 1 tasks_processing.go:69] worker 1 listening for tasks. I0327 23:01:38.393157 1 tasks_processing.go:71] worker 1 working on workload_info task. I0327 23:01:38.393166 1 tasks_processing.go:69] worker 0 listening for tasks. I0327 23:01:38.393181 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0327 23:01:38.398863 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-9.ec2.internal... I0327 23:01:38.398867 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-6.ec2.internal... I0327 23:01:38.398875 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-195.ec2.internal... I0327 23:01:38.414614 1 tasks_processing.go:74] worker 0 stopped. I0327 23:01:38.414627 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 21.420729ms to process 0 records I0327 23:01:42.833743 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 4s I0327 23:01:42.855149 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0327 23:01:42.863807 1 gather_workloads_info.go:387] No image sha256:765f0d23b637f685f98a31bd47c131b03cf72a40761a3f9a9d6320faa3c33733 (10ms) I0327 23:01:42.873176 1 gather_workloads_info.go:387] No image sha256:2904a78e2eb73fd6a9bb94c105c2a056831fb4113fbb7b0607c50adc9d879c9b (9ms) I0327 23:01:42.882653 1 gather_workloads_info.go:387] No image sha256:04c87c054a3f366a7dfbe0a93ebb0c80a098ee16842c5794b67c1202eec61996 (9ms) I0327 23:01:42.891761 1 gather_workloads_info.go:387] No image sha256:1477f3287cc75352305274f248f781c1b7c2a8696f4f7bc450c45a4a5f5b10ea (9ms) I0327 23:01:42.901687 1 gather_workloads_info.go:387] No image sha256:36b9e89c3cfcf1ab9ae500486e38afb6862cba48cb0b4d84a09508ab8f3d299f (10ms) I0327 23:01:42.910848 1 gather_workloads_info.go:387] No image sha256:25bda9d34e23ba6bc9df5cf7104bdb237269740a3a76ee025f88b34e68b7e2b5 (9ms) I0327 23:01:42.919971 1 gather_workloads_info.go:387] No image sha256:695cf2f0cc07683c2a3ce1eaf3e56fe18abc6e2bac716f7d9843f5d173b9df52 (9ms) I0327 23:01:42.928790 1 gather_workloads_info.go:387] No image sha256:2598489729a4b258e4ecda4a06f6875133f2a10ced5c5241f8a57a8a05418e36 (9ms) I0327 23:01:42.939581 1 gather_workloads_info.go:387] No image sha256:c940ea87e7d133d75ba0002ef00c0806825eed3db8094cdb260d1bac18127733 (11ms) I0327 23:01:42.956235 1 gather_workloads_info.go:387] No image sha256:1e65f9f5490fbc930d2dce96f03ddcb28990b839d7dc011ef5de5bec6c5095bf (17ms) I0327 23:01:42.965644 1 gather_workloads_info.go:387] No image sha256:90bdf9e678fe18e35401e34e1cb0912d822e9cb32b9e41cc545870540e9a87f2 (9ms) I0327 23:01:43.065063 1 gather_workloads_info.go:387] No image sha256:91828234f107c068c8a4966d08370ae7b73e637651dbc6d92c18c4553402c22c (99ms) I0327 23:01:43.164672 1 gather_workloads_info.go:387] No image sha256:101993f6e65d6fda8b90c583b718b720809182961f69bc777295f47e6df4d7ce (100ms) I0327 23:01:43.264703 1 gather_workloads_info.go:387] No image sha256:a0105d1eb62cf6ac9e5e2ef28d3e89bf6dc514bc594fc7090fe5a5ee18a09c87 (100ms) I0327 23:01:43.365615 1 gather_workloads_info.go:387] No image sha256:521712486e2c6e3c020dad6a1cb340db8e55665b69f7c208fab9cd9e965fd588 (101ms) I0327 23:01:43.465469 1 gather_workloads_info.go:387] No image sha256:0092c6ff139f8698c91349d507eb3182bc5d6c4ae040f672e347ff504accdea2 (100ms) I0327 23:01:43.565537 1 gather_workloads_info.go:387] No image sha256:0af73ad7a47c9ff53269ae2bc5163bf9e389085220c34e4bd42c92c4e7a85f6d (100ms) I0327 23:01:43.665680 1 gather_workloads_info.go:387] No image sha256:03cf4cd7ef1518610c6c7b3ad27d1622d82e98e3dc6e3f8e5d0fceb5c8d3786e (100ms) I0327 23:01:43.765244 1 gather_workloads_info.go:387] No image sha256:a258c226562adb14e3a163a1940938526ee6a0928982a7667d85d9a7334ce639 (100ms) I0327 23:01:43.865048 1 gather_workloads_info.go:398] Found image sha256:b3909bf664c77097f75b3768830863d642eed3815dab2bfb4415c771ca2d5007 (100ms) I0327 23:01:43.965131 1 gather_workloads_info.go:387] No image sha256:5a95c19d82767e0235b4edb4a0536482c816904897aae1dc3eb255cb52b87a9f (100ms) I0327 23:01:44.065044 1 gather_workloads_info.go:387] No image sha256:6e5df2aab9ce759e8f957341a11c087c233307e3c8f8760d21270efc19ccd3ce (100ms) I0327 23:01:44.164728 1 gather_workloads_info.go:387] No image sha256:4556896f77307821531ef91b7b7faccb82b824ea695693b2989f597f0deca038 (100ms) I0327 23:01:44.266027 1 gather_workloads_info.go:387] No image sha256:3d6ea9e1633f38070718a1357fcb80e6cb3932aafcc4d65e659795b186c5da15 (101ms) I0327 23:01:44.364957 1 gather_workloads_info.go:387] No image sha256:a498046d64605bcccee2440aa4f04a4602baaae263cf01d977ec5208e876b1fd (99ms) I0327 23:01:44.464762 1 gather_workloads_info.go:387] No image sha256:50197f22710766515f67944a779e00dd9ae3d17b18732d7324a970353b11f292 (100ms) I0327 23:01:44.565891 1 gather_workloads_info.go:387] No image sha256:2e57e192c3c1240fd935dcd55c8fde5e70e78bf81d6176c96edf21fafe59f8ba (101ms) I0327 23:01:44.664257 1 gather_workloads_info.go:387] No image sha256:a56211d075aa43cbb491f669a5b2e46ee023dc95b7d51dbac28f463948c5ad61 (98ms) I0327 23:01:44.764971 1 gather_workloads_info.go:387] No image sha256:653c666f842c13e0baae2e89a9b1efe0e2ef56f621ffb5b32005115d2a26ab8c (101ms) I0327 23:01:44.865359 1 gather_workloads_info.go:398] Found image sha256:32d4561d4bf1b3c4e3a7878f918f9c1422bab85bdf8711fbdf905d59692fee81 (100ms) I0327 23:01:44.965546 1 gather_workloads_info.go:387] No image sha256:7adc1eab05d6724c76ba751f6df816b08d6e70b78dee9eb94fa6fd9690542c98 (100ms) I0327 23:01:45.067465 1 gather_workloads_info.go:387] No image sha256:47154813651033d59751fb655a384dbffb64dd26f10bd7f3be0c3128d0486356 (102ms) I0327 23:01:45.164851 1 gather_workloads_info.go:387] No image sha256:76cc47f3775560477ec548c7b6fddbd99112ad6604f790697eef546576051fb7 (97ms) I0327 23:01:45.264838 1 gather_workloads_info.go:387] No image sha256:289816958633a763a72dbc44e1dad40466223164e7e253039514f0d974ea5d21 (100ms) I0327 23:01:45.364150 1 gather_workloads_info.go:387] No image sha256:74713675d580a47a56b628469c4c05c41e114e0b935960d718ff0cf7e72f613d (99ms) I0327 23:01:45.464621 1 gather_workloads_info.go:387] No image sha256:b99613494e9d6a4a8207323872d63cc02b830fc80a8bf972e62c64381aa4cd17 (100ms) I0327 23:01:45.565169 1 gather_workloads_info.go:387] No image sha256:513b1166e376feca472fdd5174f6abf0abf02925d6a21b7b4d89a2ff2fc67bed (101ms) I0327 23:01:45.664891 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (100ms) I0327 23:01:45.764982 1 gather_workloads_info.go:387] No image sha256:2e564f336c77116053f34d4201d364d8da04e789cfffa0ea422574c95f2d6404 (100ms) I0327 23:01:45.864843 1 gather_workloads_info.go:387] No image sha256:5f0b67cfbbc381243fb91ccc17345b56d05f4d717c667e8c644e5bf05633ba71 (100ms) I0327 23:01:45.964402 1 gather_workloads_info.go:387] No image sha256:943018739e3db1763c3184b460dbc409e058abbac76d57b9927faad317be85e4 (100ms) I0327 23:01:46.065292 1 gather_workloads_info.go:387] No image sha256:0a99240166165eb5718e7516a43282fe32df9c7c5e809b31b58abe44e42ff94d (101ms) I0327 23:01:46.164784 1 gather_workloads_info.go:387] No image sha256:3612c5a433effd17b7aff7ff6edf4fb27b61153908e3a0294158e4886a1d600d (99ms) I0327 23:01:46.264980 1 gather_workloads_info.go:387] No image sha256:7bd3e88d6d3f878a89203df590b684c5a77bf08df4717451d3197ab74683a80e (100ms) I0327 23:01:46.364536 1 gather_workloads_info.go:387] No image sha256:56a85660a445eced5c79a595a0eccf590087c5672d50f49d4c25ad52f9a44f04 (100ms) I0327 23:01:46.465441 1 gather_workloads_info.go:387] No image sha256:7b31223098f08328f5ddea8e5b871dbbd5f5a61ec550e8956f66793c0c6031a9 (101ms) I0327 23:01:46.564795 1 gather_workloads_info.go:387] No image sha256:1a2532940843248c57d52141185dd71fbc393ab28b65d48f682038632c1dbbad (99ms) I0327 23:01:46.564831 1 tasks_processing.go:74] worker 1 stopped. E0327 23:01:46.564841 1 gather.go:140] gatherer "workloads" function "workload_info" failed with the error: received unexpected status code 500 Internal Server Error from https://10.128.0.15:8443/gather_runtime_info I0327 23:01:46.565141 1 recorder.go:75] Recording config/workload_info with fingerprint=3deaa7a0312f7ec12c297b36fc407ad3c774b95e9b87ee863440096d1e4e73a7 I0327 23:01:46.565157 1 gather.go:177] gatherer "workloads" function "workload_info" took 8.171666099s to process 1 records E0327 23:01:46.565183 1 periodic.go:247] "Unhandled Error" err="workloads failed after 8.172s with: function \"workload_info\" failed with an error" I0327 23:01:46.566285 1 controllerstatus.go:89] name=periodic-workloads healthy=false reason=PeriodicGatherFailed message=Source workloads could not be retrieved: function "workload_info" failed with an error I0327 23:01:46.566297 1 periodic.go:209] Running conditional gatherer I0327 23:01:46.571543 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules I0327 23:01:46.677108 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer I0327 23:01:47.770163 1 conditional_gatherer.go:374] alert "KubePodNotReady" has state "pending" I0327 23:01:47.770177 1 conditional_gatherer.go:374] alert "KubePodNotReady" has state "pending" I0327 23:01:47.770180 1 conditional_gatherer.go:374] alert "KubePodNotReady" has state "pending" I0327 23:01:47.770183 1 conditional_gatherer.go:374] alert "KubeStatefulSetReplicasMismatch" has state "pending" I0327 23:01:47.770186 1 conditional_gatherer.go:374] alert "KubeStatefulSetReplicasMismatch" has state "pending" I0327 23:01:47.770188 1 conditional_gatherer.go:374] alert "KubeDaemonSetRolloutStuck" has state "pending" I0327 23:01:47.770191 1 conditional_gatherer.go:374] alert "KubeContainerWaiting" has state "pending" I0327 23:01:47.770193 1 conditional_gatherer.go:374] alert "KubeContainerWaiting" has state "pending" I0327 23:01:47.770195 1 conditional_gatherer.go:374] alert "KubeContainerWaiting" has state "pending" I0327 23:01:47.770197 1 conditional_gatherer.go:374] alert "KubeContainerWaiting" has state "pending" I0327 23:01:47.770199 1 conditional_gatherer.go:374] alert "KubeContainerWaiting" has state "pending" I0327 23:01:47.770202 1 conditional_gatherer.go:374] alert "KubeContainerWaiting" has state "pending" I0327 23:01:47.770204 1 conditional_gatherer.go:374] alert "KubeJobFailed" has state "pending" I0327 23:01:47.770206 1 conditional_gatherer.go:374] alert "KubePdbNotEnoughHealthyPods" has state "pending" I0327 23:01:47.770208 1 conditional_gatherer.go:374] alert "KubePdbNotEnoughHealthyPods" has state "pending" I0327 23:01:47.770280 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0327 23:01:47.775697 1 conditional_gatherer.go:392] cluster version is '4.20.8' I0327 23:01:47.775725 1 tasks_processing.go:45] number of workers: 3 I0327 23:01:47.775735 1 tasks_processing.go:69] worker 2 listening for tasks. I0327 23:01:47.775739 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0327 23:01:47.775747 1 tasks_processing.go:69] worker 0 listening for tasks. I0327 23:01:47.775757 1 tasks_processing.go:69] worker 1 listening for tasks. I0327 23:01:47.775765 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0327 23:01:47.775761 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0327 23:01:47.775820 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=c8d4892cd4f2eea27d57353c6055eeb530bc2d811bfe041f6984d90395255a8c I0327 23:01:47.775766 1 tasks_processing.go:74] worker 1 stopped. I0327 23:01:47.775833 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 914ns to process 1 records I0327 23:01:47.775866 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0327 23:01:47.775874 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.138µs to process 1 records I0327 23:01:47.775879 1 tasks_processing.go:74] worker 0 stopped. I0327 23:01:47.776039 1 tasks_processing.go:74] worker 2 stopped. I0327 23:01:47.776051 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 266.92µs to process 0 records I0327 23:01:47.776067 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0327 23:01:47.776083 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0327 23:01:47.799346 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=50a1cf835acd047ab83522d9d2a5ac2bbcf723461e2ffb1af74a8cf9e89d9c23 I0327 23:01:47.799483 1 diskrecorder.go:70] Writing 110 records to /var/lib/insights-operator/insights-2026-03-27-230147.tar.gz I0327 23:01:47.806192 1 diskrecorder.go:51] Wrote 110 records to disk in 6ms I0327 23:01:47.806222 1 periodic.go:278] Gathering cluster info every 2h0m0s I0327 23:01:47.806243 1 periodic.go:279] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0327 23:02:54.785326 1 diskrecorder.go:223] Found files to send: insights-2026-03-27-230147.tar.gz I0327 23:02:54.785356 1 insightsuploader.go:150] Checking archives to upload periodically every 16m24.927916059s I0327 23:02:54.785365 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0327 23:02:54.794307 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0327 23:02:54.911132 1 requests.go:87] Successfully reported id=2026-03-27T23:02:54Z x-rh-insights-request-id=1860543488c0482889fe52cfdcfc65ff, wrote=47319 I0327 23:02:54.911183 1 insightsuploader.go:187] Uploaded report successfully in 125.811007ms I0327 23:02:54.911205 1 controller.go:128] Initializing last reported time to 2026-03-27T23:02:54Z I0327 23:02:54.911220 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0327 23:02:54.911232 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0327 23:02:54.911240 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0327 23:02:54.916311 1 controller.go:512] The operator is healthy I0327 23:03:24.804027 1 controller.go:512] The operator is healthy I0327 23:03:58.697705 1 insightsreport.go:137] Pulling report from smart-proxy I0327 23:03:58.697738 1 insightsreport.go:149] Retrieving report I0327 23:03:58.703641 1 requests.go:110] Retrieving report for cluster: cb6efd4c-6882-4071-a4bb-2bc5886b18d1 I0327 23:03:58.703654 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/cb6efd4c-6882-4071-a4bb-2bc5886b18d1/reports I0327 23:03:58.706421 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/cb6efd4c-6882-4071-a4bb-2bc5886b18d1/reports I0327 23:03:58.887066 1 insightsreport.go:184] Report retrieved I0327 23:03:58.911897 1 insightsreport.go:239] Report retrieved correctly I0327 23:05:24.803796 1 controller.go:512] The operator is healthy I0327 23:06:24.778933 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0327 23:06:24.786796 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0327 23:06:24.786815 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0327 23:06:24.790573 1 secretconfigobserver.go:119] support secret does not exist I0327 23:07:24.804372 1 controller.go:512] The operator is healthy I0327 23:09:24.812222 1 controller.go:512] The operator is healthy I0327 23:11:24.780499 1 configmapobserver.go:84] configmaps "insights-config" not found I0327 23:11:24.784823 1 configmapobserver.go:84] configmaps "insights-config" not found I0327 23:11:24.790931 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0327 23:11:24.795347 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0327 23:11:24.795364 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0327 23:11:24.799700 1 secretconfigobserver.go:119] support secret does not exist I0327 23:11:24.800277 1 controller.go:512] The operator is healthy I0327 23:13:24.801636 1 controller.go:512] The operator is healthy I0327 23:15:24.801168 1 controller.go:512] The operator is healthy I0327 23:16:24.800044 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0327 23:16:24.806266 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0327 23:16:24.806288 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0327 23:16:24.809970 1 secretconfigobserver.go:119] support secret does not exist I0327 23:17:24.801424 1 controller.go:512] The operator is healthy I0327 23:19:24.801666 1 controller.go:512] The operator is healthy I0327 23:19:24.840460 1 insightsuploader.go:146] Nothing to report since 2026-03-27T23:02:54Z I0327 23:21:24.781743 1 configmapobserver.go:84] configmaps "insights-config" not found I0327 23:21:24.786083 1 configmapobserver.go:84] configmaps "insights-config" not found I0327 23:21:24.799184 1 controller.go:512] The operator is healthy I0327 23:21:24.811046 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0327 23:21:24.815240 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0327 23:21:24.815258 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0327 23:21:24.818926 1 secretconfigobserver.go:119] support secret does not exist