I0320 09:58:09.167031 1 cmd.go:253] Using service-serving-cert provided certificates I0320 09:58:09.167523 1 observer_polling.go:159] Starting file observer I0320 09:58:09.731159 1 operator.go:76] Starting insights-operator v0.0.0-master+$Format:%H$ I0320 09:58:09.731490 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0320 09:58:09.732520 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0320 09:58:09.732645 1 secure_serving.go:57] Forcing use of http/1.1 only W0320 09:58:09.732669 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0320 09:58:09.732684 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0320 09:58:09.732690 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0320 09:58:09.732697 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0320 09:58:09.732700 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0320 09:58:09.732704 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0320 09:58:09.735788 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"7eb76287-3019-4d66-8819-cebd70b544d4", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "HyperShiftOnlyDynamicResourceAllocation", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLM", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "OpenShiftPodSecurityAdmission", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}} I0320 09:58:09.735772 1 operator.go:141] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNSInstall GatewayAPI GatewayAPIController HighlyAvailableArbiter HyperShiftOnlyDynamicResourceAllocation ImageStreamImportMode ImageVolume KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS ManagedBootImagesAzure ManagedBootImagesvSphere MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages PreconfiguredUDNAddresses ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SigstoreImageVerification SigstoreImageVerificationPKI StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSDualStackInstall AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureDualStackInstall AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement CBORServingAndStorage CRDCompatibilityRequirementOperator ClientsAllowCBOR ClientsPreferCBOR ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterAPIMachineManagement ClusterAPIMachineManagementVSphere ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud EtcdBackendQuota EventTTL EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall GCPDualStackInstall ImageModeStatusReporting IngressControllerDynamicConfigurationManager InsightsConfig InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesCPMS MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutableCSINodeAllocatableCount MutatingAdmissionPolicy NewOLM NewOLMBoxCutterRuntime NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterInstall NutanixMultiSubnets OSStreams OVNObservability OnPremDNSRecords OpenShiftPodSecurityAdmission ProvisioningRequestAvailable SELinuxMount ShortCertRotation SignatureStores TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeGroupSnapshot] I0320 09:58:09.737439 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0320 09:58:09.737459 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0320 09:58:09.737498 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0320 09:58:09.737505 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0320 09:58:09.737513 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0320 09:58:09.737520 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0320 09:58:09.737758 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0320 09:58:09.738135 1 secure_serving.go:213] Serving securely on [::]:8443 I0320 09:58:09.738162 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0320 09:58:09.740872 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0320 09:58:09.740899 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0320 09:58:09.741009 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0320 09:58:09.747303 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0320 09:58:09.747343 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0320 09:58:09.751683 1 secretconfigobserver.go:119] support secret does not exist I0320 09:58:09.756094 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0320 09:58:09.760272 1 secretconfigobserver.go:119] support secret does not exist I0320 09:58:09.762847 1 recorder.go:176] Pruning old reports every 6h13m19s, max age is 288h0m0s I0320 09:58:09.771670 1 periodic.go:216] Running clusterconfig gatherer I0320 09:58:09.771687 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0320 09:58:09.771704 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0320 09:58:09.771709 1 tasks_processing.go:45] number of workers: 32 I0320 09:58:09.771728 1 tasks_processing.go:69] worker 4 listening for tasks. I0320 09:58:09.771735 1 tasks_processing.go:69] worker 31 listening for tasks. I0320 09:58:09.771739 1 tasks_processing.go:69] worker 1 listening for tasks. I0320 09:58:09.771749 1 tasks_processing.go:71] worker 1 working on authentication task. I0320 09:58:09.771750 1 tasks_processing.go:69] worker 5 listening for tasks. I0320 09:58:09.771760 1 tasks_processing.go:69] worker 6 listening for tasks. I0320 09:58:09.771757 1 tasks_processing.go:69] worker 0 listening for tasks. I0320 09:58:09.771747 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0320 09:58:09.771742 1 tasks_processing.go:71] worker 31 working on image_pruners task. I0320 09:58:09.771778 1 insightsreport.go:296] Starting report retriever I0320 09:58:09.771779 1 tasks_processing.go:69] worker 21 listening for tasks. I0320 09:58:09.771746 1 tasks_processing.go:69] worker 2 listening for tasks. I0320 09:58:09.771785 1 tasks_processing.go:69] worker 3 listening for tasks. I0320 09:58:09.771785 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0320 09:58:09.771791 1 tasks_processing.go:69] worker 22 listening for tasks. I0320 09:58:09.771783 1 tasks_processing.go:69] worker 19 listening for tasks. I0320 09:58:09.771797 1 tasks_processing.go:69] worker 28 listening for tasks. I0320 09:58:09.771797 1 tasks_processing.go:69] worker 27 listening for tasks. I0320 09:58:09.771797 1 tasks_processing.go:69] worker 7 listening for tasks. I0320 09:58:09.771804 1 tasks_processing.go:69] worker 13 listening for tasks. I0320 09:58:09.771805 1 tasks_processing.go:69] worker 23 listening for tasks. I0320 09:58:09.771809 1 tasks_processing.go:69] worker 29 listening for tasks. I0320 09:58:09.771805 1 tasks_processing.go:69] worker 14 listening for tasks. I0320 09:58:09.771814 1 tasks_processing.go:69] worker 30 listening for tasks. I0320 09:58:09.771813 1 tasks_processing.go:69] worker 15 listening for tasks. I0320 09:58:09.771813 1 tasks_processing.go:69] worker 8 listening for tasks. I0320 09:58:09.771821 1 tasks_processing.go:69] worker 17 listening for tasks. I0320 09:58:09.771823 1 tasks_processing.go:69] worker 9 listening for tasks. I0320 09:58:09.771820 1 tasks_processing.go:69] worker 16 listening for tasks. I0320 09:58:09.771736 1 tasks_processing.go:71] worker 4 working on sap_pods task. I0320 09:58:09.771827 1 tasks_processing.go:69] worker 18 listening for tasks. I0320 09:58:09.771829 1 tasks_processing.go:69] worker 24 listening for tasks. I0320 09:58:09.771836 1 tasks_processing.go:69] worker 25 listening for tasks. I0320 09:58:09.771842 1 tasks_processing.go:69] worker 10 listening for tasks. I0320 09:58:09.771844 1 tasks_processing.go:69] worker 26 listening for tasks. I0320 09:58:09.771846 1 tasks_processing.go:69] worker 11 listening for tasks. I0320 09:58:09.771851 1 tasks_processing.go:69] worker 12 listening for tasks. I0320 09:58:09.771772 1 tasks_processing.go:69] worker 20 listening for tasks. I0320 09:58:09.771949 1 tasks_processing.go:71] worker 13 working on openstack_dataplanedeployments task. I0320 09:58:09.771950 1 tasks_processing.go:71] worker 2 working on machine_autoscalers task. I0320 09:58:09.771928 1 tasks_processing.go:71] worker 14 working on operators_pods_and_events task. I0320 09:58:09.771928 1 tasks_processing.go:71] worker 29 working on openstack_controlplanes task. I0320 09:58:09.771933 1 tasks_processing.go:71] worker 19 working on silenced_alerts task. I0320 09:58:09.771937 1 tasks_processing.go:71] worker 28 working on active_alerts task. I0320 09:58:09.771940 1 tasks_processing.go:71] worker 27 working on node_features task. I0320 09:58:09.771937 1 tasks_processing.go:71] worker 22 working on nodenetworkstates task. I0320 09:58:09.771943 1 tasks_processing.go:71] worker 7 working on cost_management_metrics_configs task. I0320 09:58:09.771945 1 tasks_processing.go:71] worker 21 working on storage_cluster task. I0320 09:58:09.771946 1 tasks_processing.go:71] worker 0 working on infrastructures task. I0320 09:58:09.771954 1 tasks_processing.go:71] worker 5 working on nodenetworkconfigurationpolicies task. I0320 09:58:09.771955 1 tasks_processing.go:71] worker 3 working on service_accounts task. I0320 09:58:09.771961 1 tasks_processing.go:71] worker 18 working on container_images task. I0320 09:58:09.771960 1 tasks_processing.go:71] worker 20 working on node_logs task. I0320 09:58:09.771958 1 tasks_processing.go:71] worker 6 working on version task. I0320 09:58:09.771924 1 tasks_processing.go:71] worker 12 working on ceph_cluster task. I0320 09:58:09.771963 1 tasks_processing.go:71] worker 10 working on openshift_logging task. I0320 09:58:09.771964 1 tasks_processing.go:71] worker 15 working on storage_classes task. I0320 09:58:09.771965 1 tasks_processing.go:71] worker 26 working on machine_config_pools task. I0320 09:58:09.771968 1 tasks_processing.go:71] worker 8 working on pdbs task. I0320 09:58:09.771967 1 tasks_processing.go:71] worker 25 working on proxies task. I0320 09:58:09.771967 1 tasks_processing.go:71] worker 24 working on sap_datahubs task. I0320 09:58:09.771971 1 tasks_processing.go:71] worker 17 working on sap_config task. I0320 09:58:09.771972 1 tasks_processing.go:71] worker 23 working on schedulers task. I0320 09:58:09.771975 1 tasks_processing.go:71] worker 16 working on openstack_version task. I0320 09:58:09.771975 1 tasks_processing.go:71] worker 11 working on machines task. I0320 09:58:09.771975 1 tasks_processing.go:71] worker 9 working on cluster_apiserver task. I0320 09:58:09.771983 1 tasks_processing.go:71] worker 30 working on image_registries task. I0320 09:58:09.774968 1 tasks_processing.go:71] worker 2 working on container_runtime_configs task. I0320 09:58:09.774987 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 2.887404ms to process 0 records I0320 09:58:09.776063 1 tasks_processing.go:71] worker 29 working on openstack_dataplanenodesets task. I0320 09:58:09.776075 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 3.843141ms to process 0 records I0320 09:58:09.776251 1 tasks_processing.go:71] worker 13 working on ingress_certificates task. I0320 09:58:09.776260 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 4.24725ms to process 0 records I0320 09:58:09.776733 1 tasks_processing.go:71] worker 4 working on clusterroles task. I0320 09:58:09.776744 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 4.893071ms to process 0 records I0320 09:58:09.778528 1 tasks_processing.go:71] worker 7 working on tsdb_status task. I0320 09:58:09.778597 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 6.042339ms to process 0 records E0320 09:58:09.778658 1 gather_node_features.go:86] GatherNodeFeatures: NodeFeatures resource not found in openshift-nfd namespace (may not be installed) I0320 09:58:09.778667 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 4.647563ms to process 0 records I0320 09:58:09.778749 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 5.798975ms to process 0 records I0320 09:58:09.778765 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 6.024149ms to process 0 records I0320 09:58:09.778774 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 6.174418ms to process 0 records I0320 09:58:09.778787 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 5.293998ms to process 0 records I0320 09:58:09.778795 1 gather.go:177] gatherer "clusterconfig" function "node_features" took 6.332399ms to process 0 records I0320 09:58:09.778801 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 5.118347ms to process 0 records I0320 09:58:09.778813 1 tasks_processing.go:71] worker 10 working on mutating_webhook_configurations task. I0320 09:58:09.778721 1 tasks_processing.go:71] worker 26 working on olm_operators task. I0320 09:58:09.779017 1 tasks_processing.go:71] worker 5 working on image task. I0320 09:58:09.779172 1 tasks_processing.go:71] worker 22 working on openshift_machine_api_events task. I0320 09:58:09.779333 1 tasks_processing.go:71] worker 21 working on monitoring_persistent_volumes task. I0320 09:58:09.779370 1 tasks_processing.go:71] worker 12 working on crds task. I0320 09:58:09.779556 1 tasks_processing.go:71] worker 27 working on nodes task. I0320 09:58:09.779186 1 controller.go:129] Initializing last reported time to 0001-01-01T00:00:00Z I0320 09:58:09.779896 1 controller.go:368] The initial operator extension status is healthy I0320 09:58:09.779933 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0320 09:58:09.779976 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0320 09:58:09.780001 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0320 09:58:09.780066 1 controller.go:531] The operator is still being initialized I0320 09:58:09.780114 1 controller.go:554] The operator is healthy I0320 09:58:09.780385 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0320 09:58:09.780498 1 operator.go:328] started I0320 09:58:09.780558 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0320 09:58:09.780613 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s E0320 09:58:09.781872 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.249.33:9091: connect: connection refused I0320 09:58:09.781891 1 tasks_processing.go:71] worker 7 working on support_secret task. E0320 09:58:09.781906 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.249.33:9091: connect: connection refused I0320 09:58:09.781925 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 3.308539ms to process 0 records I0320 09:58:09.785108 1 tasks_processing.go:71] worker 24 working on metrics task. I0320 09:58:09.785127 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 10.979417ms to process 0 records E0320 09:58:09.787153 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.249.33:9091: connect: connection refused I0320 09:58:09.787174 1 tasks_processing.go:71] worker 24 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. E0320 09:58:09.787188 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.249.33:9091: connect: connection refused I0320 09:58:09.787220 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 2.049508ms to process 0 records I0320 09:58:09.794019 1 tasks_processing.go:71] worker 17 working on machine_healthchecks task. I0320 09:58:09.794032 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 19.788807ms to process 0 records I0320 09:58:09.794195 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 19.650858ms to process 0 records I0320 09:58:09.794208 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 19.071317ms to process 0 records I0320 09:58:09.794216 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 18.009604ms to process 0 records I0320 09:58:09.794224 1 tasks_processing.go:71] worker 2 working on jaegers task. E0320 09:58:09.794228 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0320 09:58:09.794237 1 gather.go:177] gatherer "clusterconfig" function "machines" took 19.666409ms to process 0 records I0320 09:58:09.794217 1 tasks_processing.go:71] worker 16 working on networks task. I0320 09:58:09.794265 1 tasks_processing.go:71] worker 29 working on oauths task. I0320 09:58:09.794275 1 tasks_processing.go:71] worker 11 working on validating_webhook_configurations task. I0320 09:58:09.794449 1 tasks_processing.go:71] worker 1 working on feature_gates task. I0320 09:58:09.794761 1 recorder.go:75] Recording config/authentication with fingerprint=211b71914c546f9f145c80790e70a76981da912007232849bf049f67312f2304 I0320 09:58:09.794778 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 22.692166ms to process 1 records I0320 09:58:09.795131 1 tasks_processing.go:71] worker 0 working on machine_sets task. I0320 09:58:09.795517 1 recorder.go:75] Recording config/infrastructure with fingerprint=103e89c1e4e4635703e2d0cfac1371939c85e1cc7331ff8168ca28c1e55e436d I0320 09:58:09.795536 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 21.769697ms to process 1 records I0320 09:58:09.795602 1 recorder.go:75] Recording config/proxy with fingerprint=e89e9f6246cfdc85fb0b5d788e5bd1e97ad8cb93ba932a051c870ecc2d7dadb4 I0320 09:58:09.795615 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 20.832828ms to process 1 records I0320 09:58:09.795623 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 21.782075ms to process 0 records I0320 09:58:09.795656 1 tasks_processing.go:71] worker 25 working on qemu_kubevirt_launcher_logs task. I0320 09:58:09.795669 1 tasks_processing.go:71] worker 20 working on dvo_metrics task. I0320 09:58:09.795682 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=0111e20560b7f54e35156cec71ee4ed9dd39fc0b1bd51fd03f83d183af1d2d38 I0320 09:58:09.795822 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 20.725103ms to process 1 records I0320 09:58:09.795689 1 tasks_processing.go:71] worker 23 working on install_plans task. I0320 09:58:09.795959 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=6a2452fca8e7422d8ab1725ca02e8e374db3d6917f15e5fe896200befad417e6 I0320 09:58:09.795959 1 tasks_processing.go:71] worker 15 working on config_maps task. I0320 09:58:09.796129 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=8fae3781d3dbede1d7bb567553dc86281f84da654a40c5dfd99d1f08eaf96426 I0320 09:58:09.796238 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 21.19258ms to process 2 records I0320 09:58:09.796334 1 tasks_processing.go:71] worker 31 working on aggregated_monitoring_cr_names task. I0320 09:58:09.796657 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=258ef3f0b59349a1dee1065ff2a3e13b6290ffb9b1aaf933aec0190534e256d1 I0320 09:58:09.796671 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 23.635073ms to process 1 records I0320 09:58:09.796753 1 recorder.go:75] Recording config/image with fingerprint=ef43cacefe84cab4d1c2bed66b24da0f2a4c9572378928a2d9bad69fe4e2c233 I0320 09:58:09.796764 1 gather.go:177] gatherer "clusterconfig" function "image" took 16.391587ms to process 1 records I0320 09:58:09.796801 1 tasks_processing.go:71] worker 5 working on pod_network_connectivity_checks task. I0320 09:58:09.796861 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=11178106a4e5f953b561a694d1283c358ecc871d89a2ed2042bad4cf5bbde9c1 I0320 09:58:09.796893 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=7ef250e31cd0efc4647ab84f8423d70bb8fdb3afda0bf8bad86f445a0da3add9 I0320 09:58:09.796916 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=ec04b03625e2e99bfd38e1026b76d2b581fc97a5853aead8a8a7b6d915a31ecd I0320 09:58:09.796938 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=7d46635e8d01ad06b1165e56847202a34ebe6b71b09eced6534675af893dc3f8 I0320 09:58:09.796976 1 tasks_processing.go:71] worker 8 working on operators task. I0320 09:58:09.796985 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=ab4d731638a920b2b0d1d963ba08c216bf6ba28aed5e5112ddbeea0d3b3d44ff I0320 09:58:09.796999 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=f33c64009af2ff95a9f4d4ce4c2b1b6d760684ea6ceefd385671cc4a122bd21a I0320 09:58:09.797026 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=15ba5926ad2c0d9fb11add6fa1fc495c806983abb75cb5b5aadf6b204e2f1f42 I0320 09:58:09.797056 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=0be15cd8e28c974c731a139ff417c4b7bdcd975372508de55d915e8339fe3bdb I0320 09:58:09.797086 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=a27d29561ea82067d2495fbba2d6adabc9b5975b6225d6c510dd80686ac3f6d3 I0320 09:58:09.797106 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=a200a1176c9ee6cb86b6345cb780db421eba25d10fd0ed2c0cfdf34b836d9cb3 I0320 09:58:09.797125 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=35da203b991a4da1ff28f41463ce7c0f8688e5dd99beb8d8a057e3d454525e9a I0320 09:58:09.797131 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 22.12274ms to process 11 records I0320 09:58:09.797175 1 tasks_processing.go:71] worker 30 working on ingress task. I0320 09:58:09.797525 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=4d49ef2b5fef42178049d72b778d44c8ae3c8bf24aef10c1185648581fd44844 I0320 09:58:09.797537 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 22.48647ms to process 1 records I0320 09:58:09.801346 1 tasks_processing.go:71] worker 14 working on overlapping_namespace_uids task. I0320 09:58:09.802070 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 29.125623ms to process 0 records I0320 09:58:09.803584 1 tasks_processing.go:71] worker 22 working on machine_configs task. I0320 09:58:09.803603 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 24.35735ms to process 0 records I0320 09:58:09.804616 1 tasks_processing.go:71] worker 17 working on lokistack task. E0320 09:58:09.804620 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0320 09:58:09.804735 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 10.497247ms to process 0 records E0320 09:58:09.804750 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0320 09:58:09.804758 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 22.716736ms to process 0 records I0320 09:58:09.804768 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 10.381098ms to process 0 records I0320 09:58:09.804778 1 tasks_processing.go:71] worker 2 working on certificate_signing_requests task. I0320 09:58:09.804821 1 tasks_processing.go:74] worker 7 stopped. I0320 09:58:09.804951 1 tasks_processing.go:74] worker 0 stopped. I0320 09:58:09.804980 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 9.802795ms to process 0 records I0320 09:58:09.805012 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 25.612236ms to process 0 records I0320 09:58:09.805030 1 tasks_processing.go:74] worker 21 stopped. I0320 09:58:09.805078 1 tasks_processing.go:74] worker 6 stopped. I0320 09:58:09.805396 1 recorder.go:75] Recording config/version with fingerprint=10994f80658313530e1fb545fa7a55a7f0caf3314664cfe4546ff33e9415a92b I0320 09:58:09.805414 1 recorder.go:75] Recording config/id with fingerprint=5c7fdb3df40049f7203f59e858310bc688b55cabaf82d7d5670beafb3c81f00a I0320 09:58:09.805422 1 gather.go:177] gatherer "clusterconfig" function "version" took 31.708037ms to process 2 records I0320 09:58:09.805656 1 tasks_processing.go:74] worker 5 stopped. E0320 09:58:09.805720 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0320 09:58:09.805762 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 8.8392ms to process 0 records I0320 09:58:09.805959 1 tasks_processing.go:74] worker 10 stopped. I0320 09:58:09.805995 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=409995b92aad76ab968d13f6813cd078f884431e1bec7c0533309408e534ec4b I0320 09:58:09.806040 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=205e3403aaacf663cae065ce6df9db776ae8d14035f220995b7fceda60a8605c I0320 09:58:09.806084 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=a6430717fd260f6e1019c3c240a2f674fea3fef1db5a78d98567997c506e7946 I0320 09:58:09.806092 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 26.921024ms to process 3 records I0320 09:58:09.806193 1 tasks_processing.go:74] worker 9 stopped. I0320 09:58:09.806252 1 recorder.go:75] Recording config/apiserver with fingerprint=3c0531c0bd0a1575138c3851c5ce08c8f1b5127c0eb8d83eb9dcf52801c5b7fc I0320 09:58:09.806263 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 31.080403ms to process 1 records I0320 09:58:09.806446 1 tasks_processing.go:74] worker 16 stopped. I0320 09:58:09.806511 1 controller.go:254] Source scaController *sca.Controller is not ready I0320 09:58:09.806527 1 controller.go:254] Source clusterTransferController *clustertransfer.Controller is not ready I0320 09:58:09.806532 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0320 09:58:09.806537 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0320 09:58:09.806541 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0320 09:58:09.806563 1 controller.go:531] The operator is still being initialized I0320 09:58:09.806575 1 controller.go:554] The operator is healthy I0320 09:58:09.806578 1 recorder.go:75] Recording config/network with fingerprint=04a2d3f5710e5b17bf35a0d7c43a8dcb41ca80e86ff1551a3e1c0d2bbcb39854 I0320 09:58:09.806589 1 gather.go:177] gatherer "clusterconfig" function "networks" took 11.776465ms to process 1 records I0320 09:58:09.806606 1 tasks_processing.go:74] worker 27 stopped. I0320 09:58:09.807031 1 recorder.go:75] Recording config/node/ip-10-0-0-174.ec2.internal with fingerprint=e40113cafba62624428df4d6625efbe89f1fa90c097713071d160d37f564b459 I0320 09:58:09.807182 1 recorder.go:75] Recording config/node/ip-10-0-1-80.ec2.internal with fingerprint=03af16b7761d30836290d290e5bc549218e327fec408e5eeb33997317652ef44 I0320 09:58:09.807343 1 recorder.go:75] Recording config/node/ip-10-0-2-72.ec2.internal with fingerprint=ea4b3c482fd5ca40a47b19a895db10b4316f175fb825cc6882e4753870447dbf I0320 09:58:09.807359 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 26.989828ms to process 3 records I0320 09:58:09.808620 1 tasks_processing.go:74] worker 17 stopped. I0320 09:58:09.808633 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 3.943762ms to process 0 records I0320 09:58:09.815645 1 tasks_processing.go:74] worker 22 stopped. E0320 09:58:09.815715 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0320 09:58:09.815782 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0320 09:58:09.815832 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 12.041322ms to process 1 records I0320 09:58:09.816101 1 tasks_processing.go:74] worker 29 stopped. I0320 09:58:09.816691 1 recorder.go:75] Recording config/oauth with fingerprint=9f815493daaefd6f82bdc53d89130d0186c26fec9660fe5a51ef54eb95490a25 I0320 09:58:09.816773 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 21.818293ms to process 1 records I0320 09:58:09.816890 1 tasks_processing.go:74] worker 4 stopped. I0320 09:58:09.816459 1 gather_logs.go:145] no pods in namespace were found I0320 09:58:09.817150 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=e88bc3d94be31693b0599cdce25705072f4ea89da22aa8f169807d0b7a83e9d5 I0320 09:58:09.817345 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=ce728581f2c3f6fa7a812297122605fa4eede397e5855b04213ee74b1128b47c I0320 09:58:09.817366 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 39.915976ms to process 2 records I0320 09:58:09.817541 1 tasks_processing.go:74] worker 1 stopped. I0320 09:58:09.817558 1 recorder.go:75] Recording config/featuregate with fingerprint=283a6f0ab3325ef91b1d1b76a9f66597a219701cd40e98faf486cc477b9b37ad I0320 09:58:09.817596 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 22.273319ms to process 1 records I0320 09:58:09.817615 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 21.402939ms to process 0 records I0320 09:58:09.817626 1 tasks_processing.go:74] worker 25 stopped. I0320 09:58:09.817742 1 tasks_processing.go:74] worker 30 stopped. I0320 09:58:09.817852 1 recorder.go:75] Recording config/ingress with fingerprint=c4f6e5af70fd6e15cedf16186c519841a9f205b4a21cae5d02b3baebec77389d I0320 09:58:09.817870 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 20.329212ms to process 1 records I0320 09:58:09.818592 1 tasks_processing.go:74] worker 14 stopped. I0320 09:58:09.818657 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0320 09:58:09.818690 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 17.187701ms to process 1 records I0320 09:58:09.818853 1 tasks_processing.go:74] worker 12 stopped. I0320 09:58:09.819218 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=d0a0a70624a08ab11fafe119691f6de1fd799f507d317e53e05ed2c6e20b87ab I0320 09:58:09.819487 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=6bd49c2bf4d1a2764f79a5021f32ea144ea97524aa1ccdd45832f96a6497eaa5 I0320 09:58:09.819502 1 gather.go:177] gatherer "clusterconfig" function "crds" took 39.191978ms to process 2 records I0320 09:58:09.819552 1 recorder.go:75] Recording config/olm_operators with fingerprint=1984636b9757927522eccbd9bba7a18c1d43d5e7bfd113c831d1ff6ef0abe759 I0320 09:58:09.819561 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 39.726105ms to process 1 records I0320 09:58:09.819568 1 tasks_processing.go:74] worker 26 stopped. I0320 09:58:09.823154 1 tasks_processing.go:74] worker 11 stopped. I0320 09:58:09.823239 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=346c620ebd1aee03f9f962d249e6a29aab52e167ac54fc5df6d14b55bd145e19 I0320 09:58:09.823296 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=b2e946804e70f0b136ecfb5bc6082074142816e3e68b6ed1569428210208656c I0320 09:58:09.823398 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=3c4c8adbb9f1334535211675dd4ff0428d4088c7f7647731685c11ddee67eacc I0320 09:58:09.823423 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=09ec09ee5624af58d875b2620774242a5f81a1101cd8898516bd2d23eff67393 I0320 09:58:09.823445 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=f51cb94c508762db6fbfa9c13afc394f5af6ddeb1eba515bc6edb4208dc54f08 I0320 09:58:09.823468 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=7d47a4677901283660843276235cb78fbc518ee9c747ab206c4f897b6f8b34fd I0320 09:58:09.823490 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=69dcd8693446bff183e87b64d0ed72b59c12fd30185c0c325e0cda8ccca6378e I0320 09:58:09.823512 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=1605020dc7f2fcd4b99d95166c37af96aa1b6ef409a8684d5974036b5e287392 I0320 09:58:09.823534 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=6ac3ababc5cb73ad9931a9c1566f949fc9a96d3b347980f41d86514f1c2680ec I0320 09:58:09.823573 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=aea2f918916ea93426e863dbc2cbaee1b371cc4a44cf4edd60cec718f0d12296 I0320 09:58:09.823610 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=2c0e377674c565afd2da490669dd80de20166d6de400122231fdf264ba089acf I0320 09:58:09.823645 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=ec37fa532af0452b6c4ba57e8e28a4ab8fc37aef97b02746400589fe0467811b I0320 09:58:09.823671 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=0b89564e8cebf9f0ce99d293a992f5496c6f665cb3f1b3c31ccb778b95b23c44 I0320 09:58:09.823679 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 28.862551ms to process 13 records W0320 09:58:09.834894 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0320 09:58:09.835102 1 tasks_processing.go:74] worker 2 stopped. I0320 09:58:09.835127 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 30.313187ms to process 0 records I0320 09:58:09.836708 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0320 09:58:09.837581 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0320 09:58:09.837590 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0320 09:58:09.837640 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0320 09:58:09.839558 1 tasks_processing.go:74] worker 13 stopped. E0320 09:58:09.839579 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0320 09:58:09.839588 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2p5cer0igbj7c319rhdi0r3gc63kvskh-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2p5cer0igbj7c319rhdi0r3gc63kvskh-primary-cert-bundle-secret" not found I0320 09:58:09.839708 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=5cd73ffdab81b20d934982929ab352169eadd0c0b776f6d6807ffaf8970192be I0320 09:58:09.839726 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 63.290825ms to process 1 records I0320 09:58:09.840936 1 tasks_processing.go:74] worker 18 stopped. I0320 09:58:09.841644 1 base_controller.go:82] Caches are synced for ConfigController I0320 09:58:09.841697 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0320 09:58:09.843341 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-5b5d9b4544-2c9n4 with fingerprint=852c7c7d3d3defc37415974e557d07142c36d01efd19ed7e4af94ba05c970cc8 I0320 09:58:09.843613 1 recorder.go:75] Recording config/pod/openshift-ovn-kubernetes/ovnkube-node-8dgxz with fingerprint=edf0d26180840daabd1574f8f88bb856bb0e8232233f03a7c0c43ba6658c0cc5 I0320 09:58:09.843829 1 recorder.go:75] Recording config/pod/openshift-ovn-kubernetes/ovnkube-node-n7cth with fingerprint=021c1d8135889ec0707ad3674a4802c684ead5cdfa5de3338280408efc79a0e3 I0320 09:58:09.843909 1 recorder.go:75] Recording config/running_containers with fingerprint=e4c45bc94314964a0e7b4fd4b77b00249859ca54761dfa31e732fe2248f54f7d I0320 09:58:09.843918 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 67.939875ms to process 4 records I0320 09:58:09.845169 1 tasks_processing.go:74] worker 31 stopped. I0320 09:58:09.845185 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 48.815914ms to process 0 records I0320 09:58:09.845380 1 tasks_processing.go:74] worker 24 stopped. I0320 09:58:09.845396 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 58.193879ms to process 0 records I0320 09:58:09.850114 1 configmapobserver.go:84] configmaps "insights-config" not found I0320 09:58:09.856266 1 tasks_processing.go:74] worker 15 stopped. E0320 09:58:09.856288 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0320 09:58:09.856300 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0320 09:58:09.856306 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0320 09:58:09.856344 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=c8a9805405aae7d04b7f03c09c51097942070877009b6daebd5573ef8127446e I0320 09:58:09.856390 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0320 09:58:09.856400 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0320 09:58:09.856406 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=3b70b9787487b548204d806ebebf90b0c2d460145e34cb33c0d8d3a7a011d907 I0320 09:58:09.856424 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0320 09:58:09.856469 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0320 09:58:09.856480 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0320 09:58:09.856488 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 60.10454ms to process 7 records I0320 09:58:09.881231 1 base_controller.go:82] Caches are synced for LoggingSyncer I0320 09:58:09.881241 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0320 09:58:09.891521 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0320 09:58:09.895431 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0320 09:58:09.938314 1 cluster_transfer.go:102] no available accepted cluster transfer I0320 09:58:09.938343 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0320 09:58:10.028071 1 sca.go:266] etc-pki-entitlement secret successfully created I0320 09:58:10.028087 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated I0320 09:58:10.827698 1 tasks_processing.go:74] worker 19 stopped. I0320 09:58:10.827718 1 recorder.go:75] Recording config/silenced_alerts.json with fingerprint=37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570 I0320 09:58:10.827729 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 1.055354086s to process 1 records I0320 09:58:10.827967 1 tasks_processing.go:74] worker 28 stopped. I0320 09:58:10.828007 1 recorder.go:75] Recording config/alerts with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 W0320 09:58:10.828023 1 gather.go:155] issue recording gatherer "clusterconfig" function "active_alerts" result "config/alerts.json" because of the warning: warning: the record with the same fingerprint "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" was already recorded at path "config/namespaces_with_overlapping_uids.json", recording another one with a different path "config/alerts.json" I0320 09:58:10.828038 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 1.055605587s to process 1 records W0320 09:58:10.831823 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0320 09:58:11.831984 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0320 09:58:12.249899 1 tasks_processing.go:74] worker 8 stopped. I0320 09:58:12.250015 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=42c8b71f1a74a1f47052c66c1a6cb3665e05a458864f20616cd7b1f74d932382 I0320 09:58:12.250063 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0320 09:58:12.250099 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=e9cf61170e7f5d6b89ca0614ec20f7efb181920ed1d393abc58982059f3aa5b5 I0320 09:58:12.250109 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0320 09:58:12.250131 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=932f3b1dae29a324fb47d183a02b632113c884f254433a1eb4258006a287f279 I0320 09:58:12.250145 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0320 09:58:12.250177 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=d44346eeaeea651d1ccd2f4e60d967c5ba7be9737cf642ab3f7e579454cffe38 I0320 09:58:12.250203 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=dfb7751b6da0625e51f8b4bb3f17ed07dfde8768d445ca9e4a2317b7167bcfc4 I0320 09:58:12.250226 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=2314684c649957905898110bbd7b3ccc20618914d1184b6a3a8ddfae93c66af8 I0320 09:58:12.250256 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=c15e2aab146e456508206b9482079ecec97f69247e57d011942e1b433357d0aa I0320 09:58:12.250522 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=d7b438c9860a2e43b40b50fb719c84960d2c9f1438b3ed7a511a5eb2f8bd7912 I0320 09:58:12.250542 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=eb7c3b7594a9a663cb719582cb4a153f9e32b3359e5c99470060275752f31967 I0320 09:58:12.250556 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0320 09:58:12.250575 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=c24fa8cb12f2e5ca1e78046159a044ace33626f732f18a196c5965f8b2b634a7 I0320 09:58:12.250586 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0320 09:58:12.250601 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=a88c3a7e0d39d386a0c5818de799933125e02296c3fa06b125551f1b8574cd48 I0320 09:58:12.250610 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0320 09:58:12.250628 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=5cc1febff9b140a31be56a51b2d55e8e6917c1d03ac38a1cf45f1faf1048c8f9 I0320 09:58:12.250637 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0320 09:58:12.250657 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=bec466d42cef33a0e2a661160d2ebe81d6730c6fd7f62e6b5c95699d23f1b2a2 I0320 09:58:12.250760 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=4e167ee9e2f319fbc18b51bc33a56a3853f9c78c434fddcd6c381eb1238fa1aa I0320 09:58:12.250770 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0320 09:58:12.250777 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0320 09:58:12.250798 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0320 09:58:12.250821 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=4f975cb520f474b567d88fdabb7d2e320395a84065fbe8c0f39343e6a1d1a638 I0320 09:58:12.250844 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=f93a9fb444d89481519538244ade6068604e7781b6707f1e2531db6aaef944f9 I0320 09:58:12.250853 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0320 09:58:12.250871 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=2348d1c172190c01d6be3bbb33e1e272c4b228fb04a08a3a68a5f33e9f0bc0eb I0320 09:58:12.250880 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0320 09:58:12.250896 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=59afd0bf2ea84f911c749595825c8cbd2e4e6d7f40f43c926f7f9b705304dd20 I0320 09:58:12.250905 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0320 09:58:12.250918 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=a4554ad3de672f59bb4c2ab8bd8d7547d40289f6955297ae7cc868620fb6b95f I0320 09:58:12.250934 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=e812b226d33806137aee18ae5c14ad307dc5edce5fd6539f13d6fa6b0bfdfee4 I0320 09:58:12.250949 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=42ca972c4bb17be2a323eedbc8bb090a404b6a250eb4689a79cee5504b0fcc56 I0320 09:58:12.250977 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=91e5d216e836ece1b485a347ce1152f97381dd22b7aad37c9c23350edfb16698 I0320 09:58:12.250989 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0320 09:58:12.251017 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=94ed6ba818d1c16d946bfbb41e7006e4c0655f714b367c8ca9c0fb6acd88302c I0320 09:58:12.251034 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0320 09:58:12.251043 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0320 09:58:12.251050 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.452897903s to process 39 records W0320 09:58:12.831131 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0320 09:58:13.832294 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0320 09:58:14.831546 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0320 09:58:14.831580 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0320 09:58:14.831595 1 tasks_processing.go:74] worker 20 stopped. E0320 09:58:14.831606 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0320 09:58:14.831617 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0320 09:58:14.831626 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.035794602s to process 1 records I0320 09:58:22.625311 1 tasks_processing.go:74] worker 23 stopped. I0320 09:58:22.625392 1 recorder.go:75] Recording config/installplans with fingerprint=f17dbfacc3bfddf27ca3b213b39495434cd4c4e9e3dbd69566ffb3845bbcf539 I0320 09:58:22.625406 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.829406883s to process 1 records I0320 09:58:23.382764 1 tasks_processing.go:74] worker 3 stopped. I0320 09:58:23.383069 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=16e4c55b4d63cefcfc7800aa646e617d46f0ec5f940c3fd1a629fc5ae985c964 I0320 09:58:23.383085 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.609804541s to process 1 records E0320 09:58:23.383149 1 periodic.go:254] "Unhandled Error" err="clusterconfig failed after 13.611s with: function \"tsdb_status\" failed with an error, function \"metrics\" failed with an error, function \"machines\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"support_secret\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machine_configs\" failed with an error, function \"ingress_certificates\" failed with an error, function \"config_maps\" failed with an error, function \"dvo_metrics\" failed with an error" I0320 09:58:23.384264 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "tsdb_status" failed with an error, function "metrics" failed with an error, function "machines" failed with an error, function "machine_healthchecks" failed with an error, function "support_secret" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machine_configs" failed with an error, function "ingress_certificates" failed with an error, function "config_maps" failed with an error, function "dvo_metrics" failed with an error I0320 09:58:23.384278 1 periodic.go:216] Running workloads gatherer I0320 09:58:23.384294 1 tasks_processing.go:45] number of workers: 2 I0320 09:58:23.384304 1 tasks_processing.go:69] worker 1 listening for tasks. I0320 09:58:23.384308 1 tasks_processing.go:71] worker 1 working on helmchart_info task. I0320 09:58:23.384337 1 tasks_processing.go:69] worker 0 listening for tasks. I0320 09:58:23.384357 1 tasks_processing.go:71] worker 0 working on workload_info task. I0320 09:58:23.391994 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-80.ec2.internal... I0320 09:58:23.392007 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-72.ec2.internal... I0320 09:58:23.392022 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-174.ec2.internal... I0320 09:58:23.418078 1 tasks_processing.go:74] worker 1 stopped. I0320 09:58:23.418097 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 33.753669ms to process 0 records I0320 09:58:28.226514 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 5s I0320 09:58:28.253141 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0320 09:58:28.266045 1 gather_workloads_info.go:387] No image sha256:5efa227ddac82e56eddc03761e220af1f2d6a4b22ee00b2baf426523f26c0319 (14ms) I0320 09:58:28.274684 1 gather_workloads_info.go:387] No image sha256:1aabd00f813e1afccdab16a845e6346288f5cf7e320fa5bb97c683702191c575 (9ms) I0320 09:58:28.283189 1 gather_workloads_info.go:387] No image sha256:bbb5af28bbed69e8189825fdc6665f4e8dbcd0c0c83fa479943ea102fe9452d6 (8ms) I0320 09:58:28.291443 1 gather_workloads_info.go:387] No image sha256:50b7c5185c4a6e63145753415c92723c59136d14b951ea0f4f06739b00893329 (8ms) I0320 09:58:28.300315 1 gather_workloads_info.go:387] No image sha256:05376e7e2304d22dd866d8e25dd1e5ff8988b2232059fadfd12cb440af785635 (9ms) I0320 09:58:28.308248 1 gather_workloads_info.go:387] No image sha256:3e262d5207d5e2077e4a14f4fbddc70d4d2651dbaaf955147013feb58a18cddd (8ms) I0320 09:58:28.317128 1 gather_workloads_info.go:387] No image sha256:1bf773bc4ec02bf6d68c82f32175366b01fb24f6da8c724b7621748692aec7dc (9ms) I0320 09:58:28.325002 1 gather_workloads_info.go:387] No image sha256:3666bcb850e1518aec86d322af52e95f725602ad0ab770d2b1e5b39b6dbde0b5 (8ms) I0320 09:58:28.332991 1 gather_workloads_info.go:387] No image sha256:3e889ea85c9c4748c548b84ae135a8adc88f42026ce8073538ca2fe7d89b8004 (8ms) I0320 09:58:28.341485 1 gather_workloads_info.go:387] No image sha256:2962c6c31d7078b34cf7ba0c9b0ea685dc539cea67b06bd38b6ac566298bb790 (8ms) I0320 09:58:28.362843 1 gather_workloads_info.go:387] No image sha256:17e68da37c74bdc95a4eb43b51ce671027807ee610a4ee5237324a8c0dc9d044 (21ms) I0320 09:58:28.460888 1 gather_workloads_info.go:387] No image sha256:11de6fe1741f5dd485e59ee2e703868badda81de0a179513be70b6ac9fc4655f (98ms) I0320 09:58:28.561777 1 gather_workloads_info.go:387] No image sha256:124f1b391b0f376a5deffd05b29ef0896ca0912e1f67b3a2051ed6678c54a57e (101ms) I0320 09:58:28.663589 1 gather_workloads_info.go:387] No image sha256:12e6985890d3c549f7cc9881892793b27277f72e2739cb910aaf9f4dd7215df1 (102ms) I0320 09:58:28.761835 1 gather_workloads_info.go:387] No image sha256:1c325fa866a794c56a7b30ca9d4fe225606566651bb12a2ecce8f60cbddb2809 (98ms) I0320 09:58:28.861580 1 gather_workloads_info.go:387] No image sha256:188e4e2d697e2ca3c7c16772c0ab12253f00bd8433d8691ed7058fe7bafbbbf4 (100ms) I0320 09:58:28.962287 1 gather_workloads_info.go:387] No image sha256:60648d27017709a420e43353f9a83d7c466017b3b771200672a4dbc3c3f7a3ae (101ms) I0320 09:58:29.062579 1 gather_workloads_info.go:387] No image sha256:30a054cfe1b8330bd57d762921ca5cd281ad2fdd7994fd3093ffadb5d767a564 (100ms) I0320 09:58:29.162260 1 gather_workloads_info.go:387] No image sha256:79fca57a4fd5ef00472a287218b84061b941905c61fcc55e89073dd93fa77648 (100ms) I0320 09:58:29.262362 1 gather_workloads_info.go:387] No image sha256:6521242b8805e70936f8e49c7d50a319adaeb73b4e5dec4693b0f93a34e397c5 (100ms) I0320 09:58:29.361736 1 gather_workloads_info.go:387] No image sha256:7db22f56cf9d223b1437e8cba3aa9e04e8ae8b973e23e85133bf1d3d8d66f15e (99ms) I0320 09:58:29.462118 1 gather_workloads_info.go:387] No image sha256:a9dc28a6962d195dcce0c76a544aad5f784dd90b98282c169d7b94c870a85f8b (100ms) I0320 09:58:29.562370 1 gather_workloads_info.go:387] No image sha256:7f426776d3ef70b585018b8d813f5eedd70373536681c6240df224ebf9ff6329 (100ms) I0320 09:58:29.663023 1 gather_workloads_info.go:387] No image sha256:03a7d45ceb838c746bdcdac1a5cc02d26273b319d9da4498bb3ee31c3e130b43 (101ms) I0320 09:58:29.763618 1 gather_workloads_info.go:387] No image sha256:3e772c41ef85aaccf63c409ee72b43d632b36d4cc66a3790835dffdcf9b7433e (101ms) I0320 09:58:29.863250 1 gather_workloads_info.go:387] No image sha256:0b2cc6aadb1abc1d35a55da0c9884eb9426d7ff7a97174daeb09bb1e1a23cfae (100ms) I0320 09:58:29.961622 1 gather_workloads_info.go:387] No image sha256:7ea1ff613255c1ee67b34ec82b68ec6f5a4a6d283620a8e0ad142d4a31eb9211 (98ms) I0320 09:58:30.088607 1 gather_workloads_info.go:387] No image sha256:8f13a1e673c882ee926e97270249a20d3c40797a8c7c5abb2fda9c98c24a2eb4 (127ms) I0320 09:58:30.174792 1 gather_workloads_info.go:398] Found image sha256:db40a3d4e50016c645fdb4a1f389dfcdbc9febc10c174816d681abe5c14153cd (86ms) I0320 09:58:30.281488 1 gather_workloads_info.go:387] No image sha256:3d68a9523f4c5c22a48dfa151bbfde78b2f360697afe3b9d8bfb53e8c359bf95 (106ms) I0320 09:58:30.362724 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (81ms) I0320 09:58:30.463963 1 gather_workloads_info.go:398] Found image sha256:670004bffddb41c787b9f9a5c7d1615453d4acfa6452dedfa61ad2f305eea814 (101ms) I0320 09:58:30.567257 1 gather_workloads_info.go:387] No image sha256:04c7a2673571e9069c934d86b4308c0f57b2715b45db500837b7708ae1f20ae7 (103ms) I0320 09:58:30.664050 1 gather_workloads_info.go:387] No image sha256:862d5befeba504d5058264cf65f16fc307eea4eb3914fbbbf40fbe61714b0958 (97ms) I0320 09:58:30.763481 1 gather_workloads_info.go:387] No image sha256:25d9148ea34149935323231c64952679727b6f7f235febc3be1df63e0bf18c5e (99ms) I0320 09:58:30.863346 1 gather_workloads_info.go:387] No image sha256:9dee9b6866db30456c277ff1f7a114dd4ce25bff8aced70218623af9423ec3cf (100ms) I0320 09:58:30.964507 1 gather_workloads_info.go:387] No image sha256:7dd9584d291967387f2a1ade991ad3bf8565bb59f49b013217951399e8273498 (101ms) I0320 09:58:31.063802 1 gather_workloads_info.go:387] No image sha256:04ee35aa6f87e954d386228808017cf2938c5a96a052059c6767078d5f3ad8d6 (99ms) I0320 09:58:31.162906 1 gather_workloads_info.go:387] No image sha256:07f14370b57046dc2ac49be933bb2248aaf5943e8fb56ce9703eaa7c673b999f (99ms) I0320 09:58:31.264136 1 gather_workloads_info.go:387] No image sha256:2af652f68b17ed49c2565928e4c101880e949c4484c6ee958793a8e668e47bea (101ms) I0320 09:58:31.395473 1 gather_workloads_info.go:387] No image sha256:317131706dcce9c87eb4e1efb8446bdbf5e0726e120a8f6c655e2acd89d60e2d (131ms) I0320 09:58:31.499482 1 gather_workloads_info.go:387] No image sha256:1ff360b712000d39cbb58158186ff0a18764278e6fa98ef9f21d113b978d9a79 (104ms) I0320 09:58:31.569868 1 gather_workloads_info.go:387] No image sha256:0c90ca6acbe2f4fdbd253ae22f5b3286247c6940321f7d2b9801aea21ef5269f (70ms) I0320 09:58:31.665645 1 gather_workloads_info.go:387] No image sha256:41e0dcd2f91df9234249f168b3fbe6d55cfdb3ebd2e598d5edfa47f905c046cb (96ms) I0320 09:58:31.762350 1 gather_workloads_info.go:387] No image sha256:11a864539ae77a94afcf8d2e1abf9168081c5081a1bc06088f4be5206cdc02df (97ms) I0320 09:58:31.862624 1 gather_workloads_info.go:387] No image sha256:876fb57ffb255ed75a452405bcd1d116788a34b19264886e6e087ff544c17083 (100ms) I0320 09:58:32.016593 1 gather_workloads_info.go:387] No image sha256:2a4f6a54cd5135c8121832236063419071fa5d00c10a051c197d3d01370e4d9c (154ms) I0320 09:58:32.016631 1 tasks_processing.go:74] worker 0 stopped. E0320 09:58:32.016645 1 gather.go:140] gatherer "workloads" function "workload_info" failed with the error: received unexpected status code 500 Internal Server Error from https://10.129.0.15:8443/gather_runtime_info I0320 09:58:32.016932 1 recorder.go:75] Recording config/workload_info with fingerprint=e0d042b3f95dcb6f7e835ec37f0d209862b7710e6f5e39bce9385d1ac35aae88 I0320 09:58:32.016952 1 gather.go:177] gatherer "workloads" function "workload_info" took 8.632265027s to process 1 records E0320 09:58:32.016984 1 periodic.go:254] "Unhandled Error" err="workloads failed after 8.632s with: function \"workload_info\" failed with an error" I0320 09:58:32.018087 1 controllerstatus.go:89] name=periodic-workloads healthy=false reason=PeriodicGatherFailed message=Source workloads could not be retrieved: function "workload_info" failed with an error I0320 09:58:32.018100 1 periodic.go:216] Running conditional gatherer I0320 09:58:32.031701 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.21.5/gathering_rules I0320 09:58:32.268047 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer E0320 09:58:32.270573 1 conditional_gatherer.go:324] unable to update alerts cache: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?match%5B%5D=ALERTS%7Balertstate%3D%22firing%22%7D&query=ALERTS": dial tcp 172.30.249.33:9091: connect: connection refused I0320 09:58:32.270664 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0320 09:58:32.288875 1 conditional_gatherer.go:392] cluster version is '4.21.5' E0320 09:58:32.288891 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0320 09:58:32.288896 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0320 09:58:32.288900 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0320 09:58:32.288903 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0320 09:58:32.288906 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0320 09:58:32.288909 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0320 09:58:32.288913 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0320 09:58:32.288915 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0320 09:58:32.288918 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0320 09:58:32.288922 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0320 09:58:32.288936 1 tasks_processing.go:45] number of workers: 3 I0320 09:58:32.288948 1 tasks_processing.go:69] worker 2 listening for tasks. I0320 09:58:32.288951 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0320 09:58:32.288958 1 tasks_processing.go:69] worker 0 listening for tasks. I0320 09:58:32.288971 1 tasks_processing.go:69] worker 1 listening for tasks. I0320 09:58:32.288984 1 tasks_processing.go:74] worker 1 stopped. I0320 09:58:32.288992 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0320 09:58:32.288992 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0320 09:58:32.289038 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=9fafdcbf373f1f367f1d69551ea59114592cce1681189be0f3dde774e0530661 I0320 09:58:32.289052 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.038µs to process 1 records I0320 09:58:32.289092 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0320 09:58:32.289101 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 2.339µs to process 1 records I0320 09:58:32.289107 1 tasks_processing.go:74] worker 0 stopped. I0320 09:58:32.289272 1 tasks_processing.go:74] worker 2 stopped. I0320 09:58:32.289288 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 272.56µs to process 0 records I0320 09:58:32.289337 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0320 09:58:32.289360 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0320 09:58:32.338746 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=0ea644b3b3df145cd67dfdfeaf316465282a7f0f356dee1bdeb79d79dca8c5e9 I0320 09:58:32.338910 1 diskrecorder.go:70] Writing 113 records to /var/lib/insights-operator/insights-2026-03-20-095832.tar.gz I0320 09:58:32.346728 1 diskrecorder.go:51] Wrote 113 records to disk in 7ms I0320 09:58:32.346764 1 periodic.go:285] Gathering cluster info every 2h0m0s I0320 09:58:32.346785 1 periodic.go:286] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0320 09:59:39.772588 1 diskrecorder.go:223] Found files to send: insights-2026-03-20-095832.tar.gz I0320 09:59:39.772616 1 insightsuploader.go:150] Checking archives to upload periodically every 15m23.144829994s I0320 09:59:39.772626 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0320 09:59:39.782412 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0320 09:59:39.970152 1 requests.go:87] Successfully reported id=2026-03-20T09:59:39Z x-rh-insights-request-id=0c7e5ef25ed948b6bca9dc8e87edfaf1, wrote=52088 I0320 09:59:39.970211 1 insightsuploader.go:187] Uploaded report successfully in 197.577124ms I0320 09:59:39.970235 1 controller.go:129] Initializing last reported time to 2026-03-20T09:59:39Z I0320 09:59:39.970253 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0320 09:59:39.970266 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0320 09:59:39.970276 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0320 09:59:39.976350 1 controller.go:554] The operator is healthy I0320 10:00:09.788710 1 controller.go:554] The operator is healthy I0320 10:00:42.620382 1 insightsreport.go:137] Pulling report from smart-proxy I0320 10:00:42.620408 1 insightsreport.go:149] Retrieving report I0320 10:00:42.627550 1 requests.go:110] Retrieving report for cluster: 34494c97-6cd9-4ed8-b867-ecd8ceb844cf I0320 10:00:42.627562 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/34494c97-6cd9-4ed8-b867-ecd8ceb844cf/reports I0320 10:00:42.630653 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/34494c97-6cd9-4ed8-b867-ecd8ceb844cf/reports I0320 10:00:42.832549 1 insightsreport.go:184] Report retrieved I0320 10:00:42.859294 1 insightsreport.go:239] Report retrieved correctly I0320 10:02:09.789333 1 controller.go:554] The operator is healthy I0320 10:03:09.761284 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0320 10:03:09.767451 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0320 10:03:09.767469 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0320 10:03:09.771897 1 secretconfigobserver.go:119] support secret does not exist I0320 10:04:09.786711 1 controller.go:554] The operator is healthy I0320 10:06:09.800016 1 controller.go:554] The operator is healthy I0320 10:08:09.755656 1 configmapobserver.go:84] configmaps "insights-config" not found I0320 10:08:09.760424 1 configmapobserver.go:84] configmaps "insights-config" not found I0320 10:08:09.772419 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0320 10:08:09.778574 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0320 10:08:09.778648 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0320 10:08:09.783504 1 secretconfigobserver.go:119] support secret does not exist I0320 10:08:09.787265 1 controller.go:554] The operator is healthy I0320 10:10:09.787932 1 controller.go:554] The operator is healthy I0320 10:12:09.795775 1 controller.go:554] The operator is healthy I0320 10:13:09.784020 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0320 10:13:09.791183 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0320 10:13:09.791210 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0320 10:13:09.796161 1 secretconfigobserver.go:119] support secret does not exist I0320 10:14:09.789704 1 controller.go:554] The operator is healthy I0320 10:15:08.116210 1 insightsuploader.go:146] Nothing to report since 2026-03-20T09:59:39Z I0320 10:16:09.790391 1 controller.go:554] The operator is healthy I0320 10:18:09.759834 1 configmapobserver.go:84] configmaps "insights-config" not found I0320 10:18:09.787274 1 controller.go:554] The operator is healthy I0320 10:18:09.797090 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0320 10:18:09.807714 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0320 10:18:09.807735 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0320 10:18:09.812092 1 secretconfigobserver.go:119] support secret does not exist