I0312 11:39:21.019834 1 cmd.go:253] Using service-serving-cert provided certificates I0312 11:39:21.020270 1 observer_polling.go:159] Starting file observer W0312 11:39:21.041056 1 builder.go:272] unable to get owner reference (falling back to namespace): replicasets.apps "insights-operator-5ff5cb4f99" is forbidden: User "system:serviceaccount:openshift-insights:operator" cannot get resource "replicasets" in API group "apps" in the namespace "openshift-insights" I0312 11:39:21.465780 1 operator.go:59] Starting insights-operator v0.0.0-master+$Format:%H$ I0312 11:39:21.466040 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0312 11:39:21.466531 1 secure_serving.go:57] Forcing use of http/1.1 only W0312 11:39:21.466554 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0312 11:39:21.466558 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0312 11:39:21.466562 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0312 11:39:21.466566 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0312 11:39:21.466568 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0312 11:39:21.466571 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0312 11:39:21.466684 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0312 11:39:21.471371 1 event.go:377] Event(v1.ObjectReference{Kind:"Namespace", Namespace:"openshift-insights", Name:"openshift-insights", UID:"", APIVersion:"v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AWSEFSDriverVolumeMetrics", "AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BareMetalLoadBalancer", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ChunkSizeMiB", "CloudDualStackNodeIPs", "ConsolePluginContentSecurityPolicy", "DisableKubeletCloudCredentialProviders", "ExternalOIDC", "GCPLabelsTags", "GatewayAPI", "GatewayAPIController", "HardwareSpeed", "IngressControllerLBSubnetsAWS", "KMSv1", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "MultiArchInstallAWS", "MultiArchInstallGCP", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "NodeDisruptionPolicy", "OnClusterBuild", "PersistentIPsForVirtualization", "PrivateHostedZoneAWS", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "VSphereDriverConfiguration", "VSphereMultiVCenters", "ValidatingAdmissionPolicy"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AutomatedEtcdBackup", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "HighlyAvailableArbiter", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "InsightsRuntimeExtractor", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "MachineAPIProviderOpenStack", "MachineConfigNodes", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PinnedImages", "PlatformOperators", "ProcMountType", "RouteAdvertisements", "SELinuxChangePolicy", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass", "VolumeGroupSnapshot"}} I0312 11:39:21.471354 1 operator.go:124] FeatureGates initialized: knownFeatureGates=[AWSEFSDriverVolumeMetrics AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BareMetalLoadBalancer BuildCSIVolumes CPMSMachineNamePrefix ChunkSizeMiB CloudDualStackNodeIPs ConsolePluginContentSecurityPolicy DisableKubeletCloudCredentialProviders ExternalOIDC GCPLabelsTags GatewayAPI GatewayAPIController HardwareSpeed IngressControllerLBSubnetsAWS KMSv1 ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles MultiArchInstallAWS MultiArchInstallGCP NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation NodeDisruptionPolicy OnClusterBuild PersistentIPsForVirtualization PrivateHostedZoneAWS RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController VSphereDriverConfiguration VSphereMultiVCenters ValidatingAdmissionPolicy AWSClusterHostedDNS AutomatedEtcdBackup BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNS GCPCustomAPIEndpoints HighlyAvailableArbiter ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather InsightsRuntimeExtractor KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController MachineAPIProviderOpenStack MachineConfigNodes MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PinnedImages PlatformOperators ProcMountType RouteAdvertisements SELinuxChangePolicy SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerification SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass VolumeGroupSnapshot] I0312 11:39:21.472220 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0312 11:39:21.472242 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0312 11:39:21.472241 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0312 11:39:21.472242 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0312 11:39:21.472255 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0312 11:39:21.472261 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0312 11:39:21.472526 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0312 11:39:21.472697 1 secure_serving.go:213] Serving securely on [::]:8443 I0312 11:39:21.472732 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" E0312 11:39:21.474437 1 event.go:359] "Server rejected event (will not retry!)" err="events is forbidden: User \"system:serviceaccount:openshift-insights:operator\" cannot create resource \"events\" in API group \"\" in the namespace \"openshift-insights\"" event="&Event{ObjectMeta:{openshift-insights.189c151b942b9a82 openshift-insights 0 0001-01-01 00:00:00 +0000 UTC map[] map[] [] [] []},InvolvedObject:ObjectReference{Kind:Namespace,Namespace:openshift-insights,Name:openshift-insights,UID:,APIVersion:v1,ResourceVersion:,FieldPath:,},Reason:FeatureGatesInitialized,Message:FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{\"AWSEFSDriverVolumeMetrics\", \"AdditionalRoutingCapabilities\", \"AdminNetworkPolicy\", \"AlibabaPlatform\", \"AzureWorkloadIdentity\", \"BareMetalLoadBalancer\", \"BuildCSIVolumes\", \"CPMSMachineNamePrefix\", \"ChunkSizeMiB\", \"CloudDualStackNodeIPs\", \"ConsolePluginContentSecurityPolicy\", \"DisableKubeletCloudCredentialProviders\", \"ExternalOIDC\", \"GCPLabelsTags\", \"GatewayAPI\", \"GatewayAPIController\", \"HardwareSpeed\", \"IngressControllerLBSubnetsAWS\", \"KMSv1\", \"ManagedBootImages\", \"ManagedBootImagesAWS\", \"MetricsCollectionProfiles\", \"MultiArchInstallAWS\", \"MultiArchInstallGCP\", \"NetworkDiagnosticsConfig\", \"NetworkLiveMigration\", \"NetworkSegmentation\", \"NodeDisruptionPolicy\", \"OnClusterBuild\", \"PersistentIPsForVirtualization\", \"PrivateHostedZoneAWS\", \"RouteExternalCertificate\", \"ServiceAccountTokenNodeBinding\", \"SetEIPForNLBIngressController\", \"VSphereDriverConfiguration\", \"VSphereMultiVCenters\", \"ValidatingAdmissionPolicy\"}, Disabled:[]v1.FeatureGateName{\"AWSClusterHostedDNS\", \"AutomatedEtcdBackup\", \"BootcNodeManagement\", \"ClusterAPIInstall\", \"ClusterAPIInstallIBMCloud\", \"ClusterMonitoringConfig\", \"ClusterVersionOperatorConfiguration\", \"DNSNameResolver\", \"DualReplica\", \"DyanmicServiceEndpointIBMCloud\", \"DynamicResourceAllocation\", \"EtcdBackendQuota\", \"EventedPLEG\", \"Example\", \"Example2\", \"ExternalOIDCWithUIDAndExtraClaimMappings\", \"GCPClusterHostedDNS\", \"GCPCustomAPIEndpoints\", \"HighlyAvailableArbiter\", \"ImageStreamImportMode\", \"IngressControllerDynamicConfigurationManager\", \"InsightsConfig\", \"InsightsConfigAPI\", \"InsightsOnDemandDataGather\", \"InsightsRuntimeExtractor\", \"KMSEncryptionProvider\", \"MachineAPIMigration\", \"MachineAPIOperatorDisableMachineHealthCheckController\", \"MachineAPIProviderOpenStack\", \"MachineConfigNodes\", \"MaxUnavailableStatefulSet\", \"MinimumKubeletVersion\", \"MixedCPUsAllocation\", \"MultiArchInstallAzure\", \"NewOLM\", \"NewOLMCatalogdAPIV1Metas\", \"NewOLMOwnSingleNamespace\", \"NewOLMPreflightPermissionChecks\", \"NodeSwap\", \"NutanixMultiSubnets\", \"OVNObservability\", \"OpenShiftPodSecurityAdmission\", \"PinnedImages\", \"PlatformOperators\", \"ProcMountType\", \"RouteAdvertisements\", \"SELinuxChangePolicy\", \"SELinuxMount\", \"ShortCertRotation\", \"SignatureStores\", \"SigstoreImageVerification\", \"SigstoreImageVerificationPKI\", \"TranslateStreamCloseWebsocketRequests\", \"UpgradeStatus\", \"UserNamespacesPodSecurityStandards\", \"UserNamespacesSupport\", \"VSphereConfigurableMaxAllowedBlockVolumesPerNode\", \"VSphereHostVMGroupZonal\", \"VSphereMultiDisk\", \"VSphereMultiNetworks\", \"VolumeAttributesClass\", \"VolumeGroupSnapshot\"}},Source:EventSource{Component:openshift-insights-operator,Host:,},FirstTimestamp:2026-03-12 11:39:21.471318658 +0000 UTC m=+0.489200707,LastTimestamp:2026-03-12 11:39:21.471318658 +0000 UTC m=+0.489200707,Count:1,Type:Normal,EventTime:0001-01-01 00:00:00 +0000 UTC,Series:nil,Action:,Related:nil,ReportingController:openshift-insights-operator,ReportingInstance:,}" W0312 11:39:21.478295 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0312 11:39:21.478319 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0312 11:39:21.478445 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0312 11:39:21.483572 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0312 11:39:21.483593 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0312 11:39:21.489196 1 secretconfigobserver.go:119] support secret does not exist I0312 11:39:21.493994 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0312 11:39:21.499372 1 secretconfigobserver.go:119] support secret does not exist I0312 11:39:21.501120 1 recorder.go:156] Pruning old reports every 5h33m55s, max age is 288h0m0s I0312 11:39:21.506481 1 periodic.go:212] Running clusterconfig gatherer I0312 11:39:21.506488 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0312 11:39:21.506512 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0312 11:39:21.506526 1 tasks_processing.go:45] number of workers: 32 I0312 11:39:21.506533 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0312 11:39:21.506538 1 insightsreport.go:296] Starting report retriever I0312 11:39:21.506544 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0312 11:39:21.506557 1 tasks_processing.go:69] worker 19 listening for tasks. I0312 11:39:21.506562 1 tasks_processing.go:69] worker 15 listening for tasks. I0312 11:39:21.506568 1 tasks_processing.go:71] worker 19 working on networks task. I0312 11:39:21.506568 1 tasks_processing.go:69] worker 20 listening for tasks. I0312 11:39:21.506562 1 tasks_processing.go:69] worker 9 listening for tasks. I0312 11:39:21.506578 1 tasks_processing.go:69] worker 10 listening for tasks. I0312 11:39:21.506575 1 tasks_processing.go:69] worker 0 listening for tasks. I0312 11:39:21.506583 1 tasks_processing.go:69] worker 5 listening for tasks. I0312 11:39:21.506585 1 tasks_processing.go:69] worker 21 listening for tasks. I0312 11:39:21.506592 1 tasks_processing.go:69] worker 11 listening for tasks. I0312 11:39:21.506592 1 tasks_processing.go:69] worker 22 listening for tasks. I0312 11:39:21.506599 1 tasks_processing.go:69] worker 23 listening for tasks. I0312 11:39:21.506599 1 tasks_processing.go:69] worker 12 listening for tasks. I0312 11:39:21.506604 1 tasks_processing.go:69] worker 6 listening for tasks. I0312 11:39:21.506607 1 tasks_processing.go:69] worker 13 listening for tasks. I0312 11:39:21.506610 1 tasks_processing.go:69] worker 24 listening for tasks. I0312 11:39:21.506601 1 tasks_processing.go:69] worker 4 listening for tasks. I0312 11:39:21.506604 1 tasks_processing.go:69] worker 2 listening for tasks. I0312 11:39:21.506617 1 tasks_processing.go:69] worker 14 listening for tasks. I0312 11:39:21.506616 1 tasks_processing.go:69] worker 3 listening for tasks. I0312 11:39:21.506626 1 tasks_processing.go:69] worker 8 listening for tasks. I0312 11:39:21.506631 1 tasks_processing.go:69] worker 26 listening for tasks. I0312 11:39:21.506635 1 tasks_processing.go:69] worker 18 listening for tasks. I0312 11:39:21.506635 1 tasks_processing.go:69] worker 16 listening for tasks. I0312 11:39:21.506642 1 tasks_processing.go:69] worker 27 listening for tasks. I0312 11:39:21.506645 1 tasks_processing.go:69] worker 29 listening for tasks. I0312 11:39:21.506647 1 tasks_processing.go:69] worker 30 listening for tasks. I0312 11:39:21.506649 1 tasks_processing.go:71] worker 9 working on machine_healthchecks task. I0312 11:39:21.506650 1 tasks_processing.go:71] worker 15 working on sap_pods task. I0312 11:39:21.506656 1 tasks_processing.go:71] worker 20 working on jaegers task. I0312 11:39:21.506661 1 tasks_processing.go:71] worker 10 working on monitoring_persistent_volumes task. I0312 11:39:21.506665 1 tasks_processing.go:71] worker 12 working on version task. I0312 11:39:21.506726 1 tasks_processing.go:71] worker 3 working on openstack_version task. I0312 11:39:21.506750 1 tasks_processing.go:71] worker 26 working on proxies task. I0312 11:39:21.506846 1 tasks_processing.go:71] worker 6 working on openstack_dataplanedeployments task. I0312 11:39:21.506655 1 tasks_processing.go:71] worker 22 working on cost_management_metrics_configs task. I0312 11:39:21.506983 1 tasks_processing.go:71] worker 4 working on sap_config task. I0312 11:39:21.507015 1 tasks_processing.go:71] worker 0 working on service_accounts task. I0312 11:39:21.507067 1 tasks_processing.go:71] worker 13 working on ceph_cluster task. I0312 11:39:21.506624 1 tasks_processing.go:69] worker 25 listening for tasks. I0312 11:39:21.506618 1 tasks_processing.go:69] worker 7 listening for tasks. I0312 11:39:21.507203 1 tasks_processing.go:71] worker 7 working on config_maps task. I0312 11:39:21.507009 1 tasks_processing.go:71] worker 5 working on nodenetworkstates task. I0312 11:39:21.506737 1 tasks_processing.go:71] worker 18 working on openshift_logging task. I0312 11:39:21.506661 1 tasks_processing.go:71] worker 23 working on machine_config_pools task. I0312 11:39:21.506561 1 tasks_processing.go:69] worker 31 listening for tasks. I0312 11:39:21.507535 1 tasks_processing.go:71] worker 31 working on openstack_dataplanenodesets task. I0312 11:39:21.506636 1 tasks_processing.go:69] worker 28 listening for tasks. I0312 11:39:21.506615 1 tasks_processing.go:69] worker 1 listening for tasks. I0312 11:39:21.506657 1 tasks_processing.go:71] worker 30 working on infrastructures task. I0312 11:39:21.506743 1 tasks_processing.go:71] worker 8 working on machine_configs task. I0312 11:39:21.506841 1 tasks_processing.go:71] worker 24 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0312 11:39:21.507150 1 tasks_processing.go:71] worker 25 working on authentication task. I0312 11:39:21.506627 1 tasks_processing.go:69] worker 17 listening for tasks. I0312 11:39:21.507937 1 tasks_processing.go:71] worker 17 working on container_images task. I0312 11:39:21.507158 1 tasks_processing.go:71] worker 21 working on pod_network_connectivity_checks task. I0312 11:39:21.507168 1 tasks_processing.go:71] worker 27 working on operators task. I0312 11:39:21.507178 1 tasks_processing.go:71] worker 11 working on nodes task. I0312 11:39:21.507173 1 tasks_processing.go:71] worker 16 working on feature_gates task. I0312 11:39:21.507175 1 tasks_processing.go:71] worker 29 working on machine_autoscalers task. I0312 11:39:21.507184 1 tasks_processing.go:71] worker 14 working on operators_pods_and_events task. I0312 11:39:21.507184 1 tasks_processing.go:71] worker 2 working on overlapping_namespace_uids task. I0312 11:39:21.507663 1 tasks_processing.go:71] worker 1 working on active_alerts task. I0312 11:39:21.507668 1 tasks_processing.go:71] worker 28 working on openstack_controlplanes task. I0312 11:39:21.510772 1 tasks_processing.go:71] worker 20 working on oauths task. I0312 11:39:21.510800 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 4.10648ms to process 0 records I0312 11:39:21.510878 1 tasks_processing.go:71] worker 9 working on image_pruners task. E0312 11:39:21.510886 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0312 11:39:21.510899 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 4.215386ms to process 0 records I0312 11:39:21.517761 1 tasks_processing.go:71] worker 29 working on clusterroles task. I0312 11:39:21.517764 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 9.492587ms to process 0 records E0312 11:39:21.517896 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0312 11:39:21.517912 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 9.822809ms to process 0 records I0312 11:39:21.517922 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 11.114715ms to process 0 records I0312 11:39:21.517931 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 10.879752ms to process 0 records I0312 11:39:21.517938 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 10.455354ms to process 0 records I0312 11:39:21.517944 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 10.520502ms to process 0 records I0312 11:39:21.517950 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 10.689017ms to process 0 records I0312 11:39:21.517956 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 11.038681ms to process 0 records I0312 11:39:21.517965 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 10.900701ms to process 0 records I0312 11:39:21.517965 1 tasks_processing.go:71] worker 18 working on machine_sets task. I0312 11:39:21.517973 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 10.340275ms to process 0 records I0312 11:39:21.517980 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 9.357936ms to process 0 records I0312 11:39:21.517986 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 10.337545ms to process 0 records I0312 11:39:21.517931 1 tasks_processing.go:71] worker 21 working on dvo_metrics task. I0312 11:39:21.518067 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 11.043274ms to process 0 records I0312 11:39:21.518194 1 tasks_processing.go:71] worker 26 working on container_runtime_configs task. I0312 11:39:21.518242 1 tasks_processing.go:71] worker 15 working on lokistack task. I0312 11:39:21.518198 1 tasks_processing.go:71] worker 22 working on storage_cluster task. I0312 11:39:21.518261 1 recorder.go:70] Recording config/proxy with fingerprint=80b1df303fca3ccd2e836365bab14f1650900e53944c2ddfa8a4bac0ee88648a I0312 11:39:21.518447 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 11.277572ms to process 1 records I0312 11:39:21.518204 1 tasks_processing.go:71] worker 31 working on validating_webhook_configurations task. I0312 11:39:21.518595 1 tasks_processing.go:71] worker 19 working on ingress task. I0312 11:39:21.518208 1 tasks_processing.go:71] worker 5 working on image task. I0312 11:39:21.518212 1 tasks_processing.go:71] worker 13 working on certificate_signing_requests task. I0312 11:39:21.518215 1 tasks_processing.go:71] worker 3 working on machines task. I0312 11:39:21.518768 1 recorder.go:70] Recording config/network with fingerprint=b927f70c3e073ca8e7b494c26e07967406f1aa88ccaa97dbf55f50ca12a0f01d I0312 11:39:21.518224 1 tasks_processing.go:71] worker 6 working on openshift_machine_api_events task. I0312 11:39:21.518227 1 tasks_processing.go:71] worker 23 working on aggregated_monitoring_cr_names task. I0312 11:39:21.518230 1 tasks_processing.go:71] worker 28 working on pdbs task. I0312 11:39:21.518282 1 tasks_processing.go:71] worker 4 working on support_secret task. I0312 11:39:21.518298 1 controller.go:119] Initializing last reported time to 2026-03-12T11:39:16Z I0312 11:39:21.519219 1 controller.go:317] The initial operator extension status is healthy I0312 11:39:21.518845 1 gather.go:177] gatherer "clusterconfig" function "networks" took 11.620136ms to process 1 records I0312 11:39:21.519255 1 controller.go:203] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0312 11:39:21.519303 1 controller.go:203] Source periodic-conditional *controllerstatus.Simple is not ready I0312 11:39:21.519312 1 controller.go:203] Source periodic-workloads *controllerstatus.Simple is not ready I0312 11:39:21.519333 1 tasks_processing.go:71] worker 30 working on metrics task. I0312 11:39:21.519338 1 controller.go:458] The operator is still being initialized I0312 11:39:21.519360 1 controller.go:481] The operator is healthy I0312 11:39:21.519972 1 recorder.go:70] Recording config/infrastructure with fingerprint=9b70f9f1e6d724552d05dc713be1d576393a94bdfa5cb2d5c60f3efccbfa7228 I0312 11:39:21.519992 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 10.824689ms to process 1 records I0312 11:39:21.520222 1 tasks_processing.go:71] worker 25 working on tsdb_status task. I0312 11:39:21.520233 1 recorder.go:70] Recording config/authentication with fingerprint=8d701c5e8be6ea4dc7448f0ceb2ac0d331b5e452ec11ac31a076f90c9f028b44 I0312 11:39:21.520320 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 10.623853ms to process 1 records I0312 11:39:21.520332 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 13.489957ms to process 0 records I0312 11:39:21.520362 1 tasks_processing.go:71] worker 10 working on cluster_apiserver task. I0312 11:39:21.520389 1 tasks_processing.go:71] worker 16 working on silenced_alerts task. I0312 11:39:21.520471 1 recorder.go:70] Recording config/featuregate with fingerprint=fe0661878e79be6fec6f17f67761ed716bcaf8e18b6b04876c17d4c62365e76c I0312 11:39:21.520484 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 12.13831ms to process 1 records I0312 11:39:21.527042 1 tasks_processing.go:71] worker 11 working on nodenetworkconfigurationpolicies task. I0312 11:39:21.527414 1 recorder.go:70] Recording config/node/ip-10-0-0-38.ec2.internal with fingerprint=89fd3552dc57f67539c6a53d0c0381ab712faf5d4c793c17a884a1d37fce33d8 I0312 11:39:21.527477 1 recorder.go:70] Recording config/node/ip-10-0-1-250.ec2.internal with fingerprint=02623afabf1fc7b5c9bd054885a47577d1bb21e736d0fce84649bb9ebe3dadfb I0312 11:39:21.527530 1 recorder.go:70] Recording config/node/ip-10-0-2-72.ec2.internal with fingerprint=cde927c2e71b94d39a6d51b2ab4d50d1cb6a9e894099747d8d7ca792837ca361 I0312 11:39:21.527539 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 18.919663ms to process 3 records I0312 11:39:21.530060 1 tasks_processing.go:71] worker 8 working on olm_operators task. E0312 11:39:21.530070 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0312 11:39:21.530112 1 recorder.go:70] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0312 11:39:21.530134 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 22.344323ms to process 1 records I0312 11:39:21.530840 1 tasks_processing.go:71] worker 15 working on mutating_webhook_configurations task. I0312 11:39:21.530922 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 12.544727ms to process 0 records I0312 11:39:21.530971 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 12.992326ms to process 0 records I0312 11:39:21.530984 1 tasks_processing.go:71] worker 18 working on crds task. I0312 11:39:21.536521 1 tasks_processing.go:71] worker 22 working on node_logs task. I0312 11:39:21.536524 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 18.185821ms to process 0 records I0312 11:39:21.538334 1 tasks_processing.go:71] worker 3 working on image_registries task. E0312 11:39:21.538384 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0312 11:39:21.538398 1 gather.go:177] gatherer "clusterconfig" function "machines" took 19.56043ms to process 0 records I0312 11:39:21.538409 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 11.313241ms to process 0 records I0312 11:39:21.538667 1 recorder.go:70] Recording config/pdbs/openshift-console/console with fingerprint=c05899289f1b36dd333aa50850b7a2558c1b78a4e05c262d47e8b0578854e50c I0312 11:39:21.538740 1 recorder.go:70] Recording config/pdbs/openshift-console/downloads with fingerprint=616ddaa56b773af7a2f57d6f740fd41aa43d97d248ec9c039ef89ba786e3c1b6 I0312 11:39:21.538793 1 recorder.go:70] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=dc55094f097deb3216eb80acfebc8666b824c9c68f402a133c76eb583e4d3a67 I0312 11:39:21.538903 1 recorder.go:70] Recording config/pdbs/openshift-ingress/router-default with fingerprint=3a9e0bf06c1da366497c9bdf3142d8896d8d4a4294ecbb1f6c52e91842f15b89 I0312 11:39:21.538963 1 recorder.go:70] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=da0530ed2f236144ce27aa7358555ab5ab2e97dded08c2ed7a72a8506d317b62 I0312 11:39:21.539028 1 recorder.go:70] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=2f41c8a49dd1087f2298fd6f4380d5e77ed06d379a4ceda90c4783ae22108ab5 I0312 11:39:21.539082 1 recorder.go:70] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=88cbece346867eec8f7c4b3bfeea194c23f9bb36fd1a946204d000a18a0b5fc3 I0312 11:39:21.539138 1 recorder.go:70] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=d7e605f5d774425dcab587d7069ee72da52f4ffcbd43e7f6410150328dc59ac6 I0312 11:39:21.539207 1 recorder.go:70] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=4ff7e39b97b628176cc4c1a02750588677c600cc0c71202924cf611ab03870fc I0312 11:39:21.539259 1 recorder.go:70] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=0df13b30f7b191c234c6cf8266a8c83f75a9fa46ee0a7b197e7c2fedd0db3336 I0312 11:39:21.539305 1 recorder.go:70] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=91a2e8652ec96f0265f4a3d9aa41e4a4898eeeaa3c1b5276de6eb156d7a2c7fb I0312 11:39:21.539334 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 19.420707ms to process 11 records I0312 11:39:21.538841 1 tasks_processing.go:71] worker 28 working on storage_classes task. I0312 11:39:21.539523 1 recorder.go:70] Recording config/apiserver with fingerprint=7595398be432260ab99dd05cd99952bee70186be78c0a26c64b972c360611da2 I0312 11:39:21.539551 1 tasks_processing.go:71] worker 10 working on sap_datahubs task. I0312 11:39:21.539561 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 18.433739ms to process 1 records I0312 11:39:21.539971 1 recorder.go:70] Recording config/oauth with fingerprint=8a97a7ec07705c412730bd78ae2c8d4c50e873ae8d88824abcd2c861bd73187a I0312 11:39:21.539983 1 tasks_processing.go:71] worker 20 working on ingress_certificates task. I0312 11:39:21.539989 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 28.488259ms to process 1 records I0312 11:39:21.538846 1 tasks_processing.go:71] worker 11 working on install_plans task. I0312 11:39:21.540172 1 recorder.go:70] Recording config/ingress with fingerprint=1d9c78db5a174298297fe5eb9e38840052f9fcff4e4ec48020802d8e0194b56d I0312 11:39:21.540211 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 20.83119ms to process 1 records I0312 11:39:21.540254 1 recorder.go:70] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0312 11:39:21.540285 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 31.481547ms to process 1 records I0312 11:39:21.541977 1 recorder.go:70] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=1fbffbe49a63ec6bb42b5637dd99eb273de49e099cb4d3cc7a02bf2a0855f8db I0312 11:39:21.542046 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 29.259472ms to process 1 records E0312 11:39:21.542078 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0312 11:39:21.542105 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 22.436442ms to process 0 records I0312 11:39:21.542134 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 23.416674ms to process 0 records I0312 11:39:21.542350 1 recorder.go:70] Recording config/image with fingerprint=e8a0e60884e498725dbe5f9fe5f41b411710c95fe6604267dacaab47ab8ac06c I0312 11:39:21.542382 1 gather.go:177] gatherer "clusterconfig" function "image" took 23.447423ms to process 1 records I0312 11:39:21.543283 1 recorder.go:70] Recording config/version with fingerprint=5df55164aff32f972e4f0092e53b061416a1b24d6c45380305119be11b0cb293 I0312 11:39:21.543344 1 tasks_processing.go:74] worker 5 stopped. I0312 11:39:21.543342 1 tasks_processing.go:74] worker 26 stopped. I0312 11:39:21.543346 1 tasks_processing.go:74] worker 4 stopped. I0312 11:39:21.543285 1 tasks_processing.go:74] worker 12 stopped. I0312 11:39:21.543403 1 recorder.go:70] Recording config/id with fingerprint=0a14b17ca08e1cb324149db86104ea967017fbb51b8fc59948bd406319d574a7 I0312 11:39:21.543477 1 gather.go:177] gatherer "clusterconfig" function "version" took 35.475876ms to process 2 records I0312 11:39:21.544331 1 tasks_processing.go:74] worker 19 stopped. I0312 11:39:21.544744 1 tasks_processing.go:74] worker 2 stopped. I0312 11:39:21.544768 1 tasks_processing.go:71] worker 9 working on schedulers task. I0312 11:39:21.545554 1 tasks_processing.go:74] worker 14 stopped. I0312 11:39:21.545634 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 37.20352ms to process 0 records I0312 11:39:21.546392 1 recorder.go:70] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=90303daa3cc9e81840b449e9154984a0fce09d33e8ad3a27dd91977128f09e22 I0312 11:39:21.546436 1 tasks_processing.go:74] worker 31 stopped. I0312 11:39:21.546476 1 recorder.go:70] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=4c94f56345e63d6e76aacbe152773e3a47665f2d31e3d2dfc4ae37e264d92d2d I0312 11:39:21.546600 1 recorder.go:70] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=3cf26ae7ebd57b1f07c52f42135d5e0189e7deb9f751d664b97c77a4417ff5ad I0312 11:39:21.546646 1 recorder.go:70] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=db1ae1f1f095be5255a63ef87778dd9d23131ffe106304dff53cc41cd90862be I0312 11:39:21.546685 1 recorder.go:70] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=7c077706e2779dbee9435a08dad2a83c523397c92647c4ea23126482993de95a I0312 11:39:21.546726 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=a58d2e3d588c7afa4711986dcc8d90b1471c2c83162853dd0507a95822a2b435 I0312 11:39:21.546778 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=9e2bd21a3b2917cadf48188bee8a2d3bf9859dbd543ed29277b0ffb3b670f2fd I0312 11:39:21.546854 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=7ee0c236ae42f58cf176697fa53f66dae3bf5a9813aff878405c8ef9fc601a3c I0312 11:39:21.546915 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=114b1d9ac69d937787542fe7d28828f9da3559791f6761005b1b41d386c6b040 I0312 11:39:21.546956 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=c139906589d9433407e20b71fc6a2e66df4d2b85edb33768ab1d25fa9d12ef0f I0312 11:39:21.546995 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=f16b65454d88804d9e3574d42379104f79b5aa5fa24ebc2f5e9a08ab70a91d61 I0312 11:39:21.547037 1 recorder.go:70] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=9d909089bda1132eacaf9d1f499906cba07181886d2b7fdf5e2ae0e155a37719 I0312 11:39:21.547050 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 27.499272ms to process 12 records I0312 11:39:21.547158 1 tasks_processing.go:74] worker 13 stopped. I0312 11:39:21.547172 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 28.457318ms to process 0 records W0312 11:39:21.547246 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0312 11:39:21.549000 1 tasks_processing.go:74] worker 10 stopped. I0312 11:39:21.549063 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 9.424978ms to process 0 records I0312 11:39:21.549098 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 30.123311ms to process 0 records I0312 11:39:21.549114 1 tasks_processing.go:74] worker 6 stopped. I0312 11:39:21.549334 1 tasks_processing.go:74] worker 29 stopped. I0312 11:39:21.549648 1 recorder.go:70] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=d1a2ceae3f1a83ecfcd7bce0dac1f115ea1fa5f42fed6470bb61554eb92d1a50 I0312 11:39:21.549791 1 recorder.go:70] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=ef8f353f09b00b231fa5679248829317b11071eb84ec09c636f6d54ca5a482de I0312 11:39:21.549831 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 31.499819ms to process 2 records I0312 11:39:21.549961 1 tasks_processing.go:74] worker 22 stopped. I0312 11:39:21.549978 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 13.374873ms to process 0 records I0312 11:39:21.550166 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0312 11:39:21.550232 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0312 11:39:21.550969 1 operator.go:287] started I0312 11:39:21.551005 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0312 11:39:21.550403 1 tasks_processing.go:74] worker 3 stopped. I0312 11:39:21.551501 1 recorder.go:70] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=ff2fcfaf87171f4aa235bbbb82951305320687ac63d547f2fd5edd1d3adc5b43 I0312 11:39:21.551676 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 12.051238ms to process 1 records I0312 11:39:21.551917 1 recorder.go:70] Recording config/storage/storageclasses/gp2-csi with fingerprint=885744400e596700f43848ce4fa721fbc1f059a92db61f2719a05006a5db3a05 I0312 11:39:21.551973 1 recorder.go:70] Recording config/storage/storageclasses/gp3-csi with fingerprint=0bc32eafd33dbd2b2e875e685752a7ef04d4053269a3e45e207b65addae4b182 I0312 11:39:21.552002 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 11.577164ms to process 2 records I0312 11:39:21.552172 1 recorder.go:70] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=9b5ee4d548f1caed920423d758e60a79908b0e85a42e1b465ca28f814c30c325 I0312 11:39:21.552236 1 recorder.go:70] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=e6adddb797248826251469b67691c36fe086a9b15fad8b26513b1572076cb96e I0312 11:39:21.552300 1 recorder.go:70] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=d6cd3c5c37df63344db2a370a0b45b8d301796daf227e018f263fc0c01aeac7c I0312 11:39:21.552326 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 20.694205ms to process 3 records I0312 11:39:21.552354 1 tasks_processing.go:74] worker 15 stopped. I0312 11:39:21.552376 1 tasks_processing.go:74] worker 28 stopped. I0312 11:39:21.554953 1 tasks_processing.go:74] worker 9 stopped. I0312 11:39:21.555067 1 recorder.go:70] Recording config/schedulers/cluster with fingerprint=a6cec3e36fa8800b8feb3fceea80d53f7bdeb085483c6fe64d763d3f9e24de8d I0312 11:39:21.555082 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 10.106972ms to process 1 records I0312 11:39:21.555164 1 tasks_processing.go:74] worker 18 stopped. I0312 11:39:21.555632 1 recorder.go:70] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=5d9c015fe2a66e59d9aa73573e2ea23c1a9240b34acae1a1635c84511b508550 I0312 11:39:21.555882 1 recorder.go:70] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=a920a5e77343faf728678ccba45edff7ed365f9c4a3e93e52bd764d9070b543b I0312 11:39:21.555898 1 gather.go:177] gatherer "clusterconfig" function "crds" took 24.085408ms to process 2 records I0312 11:39:21.556754 1 tasks_processing.go:74] worker 23 stopped. I0312 11:39:21.556769 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 37.760955ms to process 0 records I0312 11:39:21.563264 1 controller.go:203] Source scaController *sca.Controller is not ready I0312 11:39:21.563286 1 controller.go:203] Source clusterTransferController *clustertransfer.Controller is not ready I0312 11:39:21.563292 1 controller.go:203] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0312 11:39:21.563298 1 controller.go:203] Source periodic-conditional *controllerstatus.Simple is not ready I0312 11:39:21.563304 1 controller.go:203] Source periodic-workloads *controllerstatus.Simple is not ready I0312 11:39:21.563325 1 controller.go:458] The operator is still being initialized I0312 11:39:21.563333 1 controller.go:481] The operator is healthy I0312 11:39:21.563368 1 tasks_processing.go:74] worker 8 stopped. I0312 11:39:21.563489 1 recorder.go:70] Recording config/olm_operators with fingerprint=3cc29910b2fce288512263e1d09b6615c1a5d8fd4f9abf00e275d6ca1cde9161 I0312 11:39:21.563545 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 33.292766ms to process 1 records I0312 11:39:21.563996 1 tasks_processing.go:74] worker 17 stopped. I0312 11:39:21.566521 1 recorder.go:70] Recording config/pod/openshift-console-operator/console-operator-85b84f584d-cgc9n with fingerprint=ee6c49997f71b8736fe2618377058e7cbea288a722be404a56d02792f8ef2412 I0312 11:39:21.566657 1 recorder.go:70] Recording config/running_containers with fingerprint=26428855d71d99ac329a9239ca7e1a39c3d31f6fe2ed7a210309d4baeb810c53 I0312 11:39:21.566676 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 56.041129ms to process 2 records I0312 11:39:21.566695 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 58.866742ms to process 0 records I0312 11:39:21.566707 1 tasks_processing.go:74] worker 24 stopped. I0312 11:39:21.572784 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0312 11:39:21.572851 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0312 11:39:21.572926 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0312 11:39:21.578922 1 base_controller.go:82] Caches are synced for ConfigController I0312 11:39:21.578934 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0312 11:39:21.582289 1 tasks_processing.go:74] worker 7 stopped. E0312 11:39:21.582307 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0312 11:39:21.582315 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0312 11:39:21.582320 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0312 11:39:21.582330 1 recorder.go:70] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=295a2bc7810c628251f1f312c209b5eb4da888326706c635ac0c12aec0969e17 I0312 11:39:21.582360 1 recorder.go:70] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0312 11:39:21.582371 1 recorder.go:70] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0312 11:39:21.582377 1 recorder.go:70] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=fe7abfa7f7aea852d8bca6b7df9b4d5de32045254ff2774f9bd30f0b6dcb7dc4 I0312 11:39:21.582386 1 recorder.go:70] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0312 11:39:21.582461 1 recorder.go:70] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0312 11:39:21.582475 1 recorder.go:70] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0312 11:39:21.582483 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 75.045732ms to process 7 records I0312 11:39:21.584129 1 configmapobserver.go:84] configmaps "insights-config" not found I0312 11:39:21.588063 1 tasks_processing.go:74] worker 20 stopped. E0312 11:39:21.588078 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0312 11:39:21.588084 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2p055t8osssausu034arnp0jmbopcfps-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2p055t8osssausu034arnp0jmbopcfps-primary-cert-bundle-secret" not found I0312 11:39:21.588143 1 recorder.go:70] Recording aggregated/ingress_controllers_certs with fingerprint=c44ac7d3d009ae922fb25049a09f54284c84d1114446e0565a147df30f9882fc I0312 11:39:21.588155 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 48.061923ms to process 1 records I0312 11:39:21.629977 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0312 11:39:21.652032 1 base_controller.go:82] Caches are synced for LoggingSyncer I0312 11:39:21.652047 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0312 11:39:21.677052 1 cluster_transfer.go:102] no available accepted cluster transfer I0312 11:39:21.677070 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0312 11:39:21.810589 1 sca.go:266] etc-pki-entitlement secret successfully created I0312 11:39:21.810608 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated I0312 11:39:22.371325 1 gather_cluster_operators.go:184] Unable to get operatorpkis.network.operator.openshift.io resource due to: operatorpkis.network.operator.openshift.io "ovn" is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot get resource "operatorpkis" in API group "network.operator.openshift.io" in the namespace "openshift-ovn-kubernetes" W0312 11:39:22.542013 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0312 11:39:22.552084 1 tasks_processing.go:74] worker 16 stopped. I0312 11:39:22.552104 1 recorder.go:70] Recording config/silenced_alerts.json with fingerprint=37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570 I0312 11:39:22.552115 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 1.031673563s to process 1 records I0312 11:39:22.552371 1 tasks_processing.go:74] worker 1 stopped. I0312 11:39:22.552414 1 recorder.go:70] Recording config/alerts with fingerprint=b82ca7b5fa71fdb5da6f7a79929e4035310cb7c59c5793d324987ba18796ddbe I0312 11:39:22.552431 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 1.04387861s to process 1 records I0312 11:39:22.567637 1 tasks_processing.go:74] worker 30 stopped. I0312 11:39:22.567683 1 recorder.go:70] Recording config/metrics with fingerprint=5eacc9523abee488eb8fbc6eca9ab5f1431cffd9186fc377fc315933043109ea I0312 11:39:22.567696 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 1.048281997s to process 1 records I0312 11:39:22.570178 1 gather_cluster_operators.go:184] Unable to get operatorpkis.network.operator.openshift.io resource due to: operatorpkis.network.operator.openshift.io "signer" is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot get resource "operatorpkis" in API group "network.operator.openshift.io" in the namespace "openshift-ovn-kubernetes" I0312 11:39:22.572342 1 tasks_processing.go:74] worker 25 stopped. I0312 11:39:22.572373 1 recorder.go:70] Recording config/tsdb.json with fingerprint=155a4e76287550723e2187df9a09f22d3d716e9591bae6b20f3b47ebf8ed9e6b I0312 11:39:22.572386 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 1.052048172s to process 1 records W0312 11:39:23.543431 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0312 11:39:23.975111 1 tasks_processing.go:74] worker 27 stopped. I0312 11:39:23.975179 1 recorder.go:70] Recording config/clusteroperator/console with fingerprint=01923975ce680daf14ebfa545ce59e384ae292d96141eb9ce25f9f275033ef22 I0312 11:39:23.975236 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0312 11:39:23.975257 1 recorder.go:70] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=ef0493586bc0f2ef97307a054b3547d4cba98f56abc7748d05d6dd4611f3300b I0312 11:39:23.975267 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0312 11:39:23.975288 1 recorder.go:70] Recording config/clusteroperator/dns with fingerprint=010e683efc74840546086187520758ea7a9c5a8075ad6f2963f14454e915b266 I0312 11:39:23.975304 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0312 11:39:23.975327 1 recorder.go:70] Recording config/clusteroperator/image-registry with fingerprint=bc9a5d2d041d8526c366527b99a40fbcd56c3cee6040650560aaf4ed48b1681c I0312 11:39:23.975354 1 recorder.go:70] Recording config/clusteroperator/ingress with fingerprint=ca361849e12aa79fa501dd3b136300d591615c23a0a2777bf90524a486bc222c I0312 11:39:23.975379 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=f067e689ec5651ff33aaad1fbab6d389d4deccc0e68e30d9380a210b82fd9424 I0312 11:39:23.975416 1 recorder.go:70] Recording config/clusteroperator/insights with fingerprint=c41a394642cde82300d625cae1e69d9de5d144adb9a787820aaf08da5bd6e907 I0312 11:39:23.975703 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=56a3a17d8394826b5fd799b381753bb82e72aef87f18ebe51c98491614435624 I0312 11:39:23.975724 1 recorder.go:70] Recording config/clusteroperator/kube-apiserver with fingerprint=d480c041d252fb8e71e259bdb30ce21f00bfad7d8256c1c7d5141dd96b65cd63 I0312 11:39:23.975732 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0312 11:39:23.975748 1 recorder.go:70] Recording config/clusteroperator/kube-controller-manager with fingerprint=611da78ea1410f82d1c7d1534c6ff66e1eb6e0f0a1fae4c75989fe38555fc2ec I0312 11:39:23.975760 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0312 11:39:23.975777 1 recorder.go:70] Recording config/clusteroperator/kube-scheduler with fingerprint=dcc09f1c6999e7c641a9c235a7805423b9bb8de1b6f718000df049c5ef5aeb92 I0312 11:39:23.975787 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0312 11:39:23.975808 1 recorder.go:70] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=2d33995b8af946d58f7bf64d622d06c593a29ec44d34a3f8f8952169d485ee55 I0312 11:39:23.975833 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0312 11:39:23.975857 1 recorder.go:70] Recording config/clusteroperator/monitoring with fingerprint=ffa6cbeeefab33c800a10b68860074b39548bb7c9d0e0b5dea769973ee80eeba I0312 11:39:23.975959 1 recorder.go:70] Recording config/clusteroperator/network with fingerprint=0c2f3993a25955cd56e1742637fab8a90a7c48198fcc3b7ea0cf93247a126ff7 I0312 11:39:23.975985 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0312 11:39:23.976007 1 recorder.go:70] Recording config/clusteroperator/node-tuning with fingerprint=faba59a559cb2c66f3b48950267598dc5a6156b2ea35c57e71e21f8f83c738e5 I0312 11:39:23.976033 1 recorder.go:70] Recording config/clusteroperator/openshift-apiserver with fingerprint=ca4968d97d2e385a62684a71181b6c41f57e08feeeb5482d587564fe2fe2927c I0312 11:39:23.976043 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0312 11:39:23.976060 1 recorder.go:70] Recording config/clusteroperator/openshift-controller-manager with fingerprint=e89b9c76e02ebf461e5065c39309dda64fcc023e9d04aa3d989ce9d793f7356b I0312 11:39:23.976072 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0312 11:39:23.976090 1 recorder.go:70] Recording config/clusteroperator/openshift-samples with fingerprint=bc384be6bf53a9af44576b4ff67ba4266001ae9ed615502ce3a9086a48d2ee7b I0312 11:39:23.976100 1 recorder.go:70] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0312 11:39:23.976114 1 recorder.go:70] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=86c281fa64507a577179118513c410a4e2bac4b395373ea4b19fc8400c1a8cb2 I0312 11:39:23.976130 1 recorder.go:70] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=625bbe0700ba280b7cc491a0741c80640c8aa7226be8cd6992bb6cdac3a26053 I0312 11:39:23.976146 1 recorder.go:70] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=a75e8d28ae1d7dd353e74a4c22ccdd9555851507fa29ccd66d4572c95fa0e71c I0312 11:39:23.976169 1 recorder.go:70] Recording config/clusteroperator/service-ca with fingerprint=7ee712696bee355b882a7d7f45912628351b84d4a78169f0c9df64e16b2cded9 I0312 11:39:23.976179 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0312 11:39:23.976202 1 recorder.go:70] Recording config/clusteroperator/storage with fingerprint=6fcad2f7140b31b4f9d9d3b63a1057f0c3e2064b25f157140068a6a361e199a2 I0312 11:39:23.976221 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0312 11:39:23.976232 1 recorder.go:70] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0312 11:39:23.976239 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.467060098s to process 37 records W0312 11:39:24.542282 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0312 11:39:25.543187 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0312 11:39:26.540414 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0312 11:39:26.540443 1 tasks_processing.go:74] worker 21 stopped. E0312 11:39:26.540457 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0312 11:39:26.540473 1 recorder.go:70] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0312 11:39:26.540487 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.022435265s to process 1 records I0312 11:39:34.432206 1 tasks_processing.go:74] worker 11 stopped. I0312 11:39:34.432263 1 recorder.go:70] Recording config/installplans with fingerprint=f17dbfacc3bfddf27ca3b213b39495434cd4c4e9e3dbd69566ffb3845bbcf539 I0312 11:39:34.432291 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.892142072s to process 1 records I0312 11:39:35.117904 1 tasks_processing.go:74] worker 0 stopped. I0312 11:39:35.118201 1 recorder.go:70] Recording config/serviceaccounts with fingerprint=73582af525bf551f979ef96bfb3cbb0257826668d63659ccecb8b26ded60e307 I0312 11:39:35.118222 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.610829317s to process 1 records E0312 11:39:35.118281 1 periodic.go:250] "Unhandled Error" err="clusterconfig failed after 13.611s with: function \"machine_healthchecks\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machine_configs\" failed with an error, function \"machines\" failed with an error, function \"support_secret\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0312 11:39:35.119387 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "machine_healthchecks" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machine_configs" failed with an error, function "machines" failed with an error, function "support_secret" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0312 11:39:35.119405 1 periodic.go:212] Running workloads gatherer I0312 11:39:35.119422 1 tasks_processing.go:45] number of workers: 2 I0312 11:39:35.119439 1 tasks_processing.go:69] worker 1 listening for tasks. I0312 11:39:35.119444 1 tasks_processing.go:71] worker 1 working on workload_info task. I0312 11:39:35.119453 1 tasks_processing.go:69] worker 0 listening for tasks. I0312 11:39:35.119516 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0312 11:39:35.149747 1 tasks_processing.go:74] worker 0 stopped. I0312 11:39:35.149764 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 30.211098ms to process 0 records I0312 11:39:35.181781 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-38.ec2.internal... I0312 11:39:35.181782 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-250.ec2.internal... I0312 11:39:35.181828 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-72.ec2.internal... I0312 11:39:39.186916 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 4s I0312 11:39:39.214289 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0312 11:39:39.223701 1 gather_workloads_info.go:387] No image sha256:a7c71f3c9714cf1717c45e12ec817be6d6209f00989e6a31d634e527f0c8147d (11ms) I0312 11:39:39.234005 1 gather_workloads_info.go:387] No image sha256:bae7a33f8db8a3d4b3c4c05498aba85a0ce463f85322067c48e79663710e616e (10ms) I0312 11:39:39.243688 1 gather_workloads_info.go:387] No image sha256:2fac754deaeade3456361eed52e344318ff16d04819384432759f0ea35530114 (10ms) I0312 11:39:39.253027 1 gather_workloads_info.go:387] No image sha256:9697bc2258bdfa9ae8c1866cc7eb0b3b46851998db827b121bdf77417a881eb3 (9ms) I0312 11:39:39.262947 1 gather_workloads_info.go:387] No image sha256:929a4a9c96a08b57fe4c315deca55c30d1990c83e34b8b5692f7646d29f715c8 (10ms) I0312 11:39:39.272673 1 gather_workloads_info.go:387] No image sha256:420c12f61c53e54eab2d99476759c0de339fca98ce0a3a782bc6545cc0e97a9c (10ms) I0312 11:39:39.284556 1 gather_workloads_info.go:387] No image sha256:0a92dd43975f972e3dc707fb37854e046813531553038147e9c90d54a8d9df73 (12ms) I0312 11:39:39.294451 1 gather_workloads_info.go:387] No image sha256:2d077e73dd76873ee2a1583aa3b3e76a1b408737f0af2e22b2aa055604d89e81 (10ms) I0312 11:39:39.304599 1 gather_workloads_info.go:398] Found image sha256:23c4fe84047a4e6cbe7f75f470e7d6fe0e61e7910dd17945b1b61bc4b72f3f2a (10ms) I0312 11:39:39.316358 1 gather_workloads_info.go:387] No image sha256:76f820bd9bd138d29305d545d7d49bfe63e2923f1fb1ec2e8eac81a388359024 (12ms) I0312 11:39:39.325783 1 gather_workloads_info.go:387] No image sha256:4f175ee49f51dc4379e9993fecd1657c7a9c4c3dc096772cb198d0212b9eea47 (9ms) I0312 11:39:39.423589 1 gather_workloads_info.go:387] No image sha256:752be69b2262be713df12c47f4bac8c2dafed272c401e9a89f8060f053d68054 (98ms) I0312 11:39:39.523285 1 gather_workloads_info.go:387] No image sha256:7854174b4b98e9b10d9c8e9b43113df1217ab21db4b2395d9399af0c3d7911a3 (100ms) I0312 11:39:39.623366 1 gather_workloads_info.go:387] No image sha256:527ed405bf66234718321996dacf51f91c64c746de9f1509416ef6ce89c90a50 (100ms) I0312 11:39:39.723399 1 gather_workloads_info.go:387] No image sha256:7941b5d9a758b8667e99cd9236f7f96ec036af61df5cbc96e16077d36700d7c7 (100ms) I0312 11:39:39.823185 1 gather_workloads_info.go:387] No image sha256:e40bbab1e52f580ab471bd7ab19ae23a4bcbaf1893243a05238af6bede7ba9cc (100ms) I0312 11:39:39.922861 1 gather_workloads_info.go:387] No image sha256:46c2c396ca651699738c3f26b79b10af2922123dd27c2e32382d2a31a87e92c7 (100ms) I0312 11:39:40.022924 1 gather_workloads_info.go:387] No image sha256:1cc2f3c83068c8c67d9070f67677e1bb6b4feff7ac5744cad08711893e56b55f (100ms) I0312 11:39:40.122663 1 gather_workloads_info.go:387] No image sha256:5cba745f09f3bd6b8028c6773597cbb30bf3639a84f2b4b6493fe20642339a2c (100ms) I0312 11:39:40.223704 1 gather_workloads_info.go:387] No image sha256:6d6ebff54b8adac74f4d1b12ac8aa0f16cd7b28370e0d6aa847d1c457e03a5b6 (101ms) I0312 11:39:40.323582 1 gather_workloads_info.go:387] No image sha256:8b1e32ce43eb1247b0558230b6e0baa85ac62f1a5e2a2089d40b4fea90538529 (100ms) I0312 11:39:40.423239 1 gather_workloads_info.go:387] No image sha256:944d9261ba7a143131fe8267c172defc5f37acc1cea3d4d373ec6fc5d8bfcc31 (100ms) I0312 11:39:40.523913 1 gather_workloads_info.go:387] No image sha256:6277dac65decba0ea094146bb99600f479eaf804696842e724de1f1b436f9a38 (101ms) I0312 11:39:40.623095 1 gather_workloads_info.go:387] No image sha256:48404bd61d05dc738ccce4d22e36b30dbcdc6015b06b4afb604fd1baeee35bf2 (99ms) I0312 11:39:40.723519 1 gather_workloads_info.go:387] No image sha256:8ef4bdf07dda423fab73484dbc66d527ce8ba8cc2d4f99210bc1bc24ea08c0cb (100ms) I0312 11:39:40.823784 1 gather_workloads_info.go:387] No image sha256:38dd36da63b1f01316a02daf6c18b821318c6ec313712147292d74abf955e0da (100ms) I0312 11:39:40.923556 1 gather_workloads_info.go:387] No image sha256:68d8e92046969bcfcbca2d2b3dcf4dcad5f6edc5fcb44861eb22f8079a43ea4d (100ms) I0312 11:39:41.022780 1 gather_workloads_info.go:387] No image sha256:bcca5c17371caa2dce7e28da4c9fe30163c340fa172187f94db90ea1bd6ab80d (99ms) I0312 11:39:41.123236 1 gather_workloads_info.go:387] No image sha256:1d471ad866b8564b209214d7dd3a7363c47e9535211d660b63b46abb278af7ab (100ms) I0312 11:39:41.223058 1 gather_workloads_info.go:387] No image sha256:3d0634fe58641d5242649c44ddba70ed67fdd6d2dcb5c2261df5cee8b33de9fd (100ms) I0312 11:39:41.323150 1 gather_workloads_info.go:387] No image sha256:1f2aa1e1026570162984f7fbb634635bcf319e408cafa8f9034ee842a2f193ab (100ms) I0312 11:39:41.423143 1 gather_workloads_info.go:387] No image sha256:87efe06afd1f04426fca7f86c0f74c4ee75c311ba199ddabd5b849b877bc59fa (100ms) I0312 11:39:41.523656 1 gather_workloads_info.go:387] No image sha256:799f5628c18e77a49c08645c03179046b5200c8cbcfd65737fb59668132f393e (100ms) I0312 11:39:41.622720 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (99ms) I0312 11:39:41.723707 1 gather_workloads_info.go:387] No image sha256:373614619b9420b110d1508c5f17e066ce69c4c226fd04d02fbb959d9ba41eb6 (101ms) I0312 11:39:41.824120 1 gather_workloads_info.go:387] No image sha256:00f977c2275f5bf03d423911159a39ca44fabc1862ecb7ae492198ec8c46c6bb (100ms) I0312 11:39:41.923087 1 gather_workloads_info.go:387] No image sha256:18956d5fbc8d0369a26cb1e00eb5c3aff1973ae8cbd3983920a0cf2851f05358 (99ms) I0312 11:39:42.023493 1 gather_workloads_info.go:387] No image sha256:039479f355d6903e1018abf8427fece3465c4348d4de2144ec38114b51bec27e (100ms) I0312 11:39:42.124001 1 gather_workloads_info.go:387] No image sha256:ce3d8211c0bde5788a4fd04c3eff31ae591437ceab2b3168dbbdfe408673c8b1 (100ms) I0312 11:39:42.223014 1 gather_workloads_info.go:387] No image sha256:6c3c7f77d7a3815bbb69cad5f6f1742c17d9597bc135ba3d116c9736d98fae33 (99ms) I0312 11:39:42.323175 1 gather_workloads_info.go:398] Found image sha256:59db91eadf1e1e5343777be43ff25fb712f300f9341be48ba05f45270944536f (100ms) I0312 11:39:42.423134 1 gather_workloads_info.go:387] No image sha256:a3d34d8f38a540ce79d34a5248b1965346155497db57b72cb4758d659418059c (100ms) I0312 11:39:42.523295 1 gather_workloads_info.go:387] No image sha256:824a3b8f78e19aa21a9f6444aefe8e0b624886cca18fe828b13acbab55e6e868 (100ms) I0312 11:39:42.623179 1 gather_workloads_info.go:387] No image sha256:14703f73bb1ca69ea03b726e340b3b68f8e294e1f22c80d1a16666ea3d4a88a3 (100ms) I0312 11:39:42.723497 1 gather_workloads_info.go:387] No image sha256:0a8a0473029ace3adbb66f490dbe560f0e7782a38cdf32c1f7dd3e092e1d191e (100ms) I0312 11:39:42.723534 1 tasks_processing.go:74] worker 1 stopped. E0312 11:39:42.723544 1 gather.go:140] gatherer "workloads" function "workload_info" failed with the error: received unexpected status code 500 Internal Server Error from https://10.129.0.8:8443/gather_runtime_info I0312 11:39:42.723871 1 recorder.go:70] Recording config/workload_info with fingerprint=6a3c45f986c52891dd5fafd72b850020d764faa90f2c2c317b4d42827e402835 I0312 11:39:42.723891 1 gather.go:177] gatherer "workloads" function "workload_info" took 7.604082167s to process 1 records E0312 11:39:42.723915 1 periodic.go:250] "Unhandled Error" err="workloads failed after 7.604s with: function \"workload_info\" failed with an error" I0312 11:39:42.725015 1 controllerstatus.go:89] name=periodic-workloads healthy=false reason=PeriodicGatherFailed message=Source workloads could not be retrieved: function "workload_info" failed with an error I0312 11:39:42.725027 1 periodic.go:212] Running conditional gatherer I0312 11:39:42.734072 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.19.9/gathering_rules I0312 11:39:43.030921 1 conditional_gatherer.go:336] updating alerts cache for conditional gatherer I0312 11:39:43.032924 1 conditional_gatherer.go:382] updating version cache for conditional gatherer I0312 11:39:43.041698 1 conditional_gatherer.go:390] cluster version is '4.19.9' I0312 11:39:43.041726 1 tasks_processing.go:45] number of workers: 3 I0312 11:39:43.041733 1 tasks_processing.go:69] worker 2 listening for tasks. I0312 11:39:43.041738 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0312 11:39:43.041750 1 tasks_processing.go:69] worker 0 listening for tasks. I0312 11:39:43.041761 1 tasks_processing.go:69] worker 1 listening for tasks. I0312 11:39:43.041766 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0312 11:39:43.041770 1 tasks_processing.go:74] worker 1 stopped. I0312 11:39:43.041762 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0312 11:39:43.041864 1 recorder.go:70] Recording insights-operator/conditional-gatherer-rules with fingerprint=c8d4892cd4f2eea27d57353c6055eeb530bc2d811bfe041f6984d90395255a8c I0312 11:39:43.041882 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.385µs to process 1 records I0312 11:39:43.041926 1 recorder.go:70] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0312 11:39:43.041938 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 862ns to process 1 records I0312 11:39:43.041944 1 tasks_processing.go:74] worker 0 stopped. I0312 11:39:43.042106 1 tasks_processing.go:74] worker 2 stopped. I0312 11:39:43.042119 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 333.172µs to process 0 records I0312 11:39:43.042134 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=AsExpected message= I0312 11:39:43.042182 1 recorder.go:70] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0312 11:39:43.069919 1 recorder.go:70] Recording insights-operator/gathers with fingerprint=e9f8cc7e76c6a52ac3770ba5456e846041b3bec9fbab2ebb95221a6f63bfba32 I0312 11:39:43.070056 1 diskrecorder.go:70] Writing 110 records to /var/lib/insights-operator/insights-2026-03-12-113943.tar.gz I0312 11:39:43.077075 1 diskrecorder.go:51] Wrote 110 records to disk in 7ms I0312 11:39:43.077108 1 periodic.go:281] Gathering cluster info every 2h0m0s I0312 11:39:43.077123 1 periodic.go:282] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0312 11:40:51.506907 1 diskrecorder.go:223] Found files to send: insights-2026-03-12-113943.tar.gz I0312 11:40:51.506941 1 insightsuploader.go:150] Checking archives to upload periodically every 15m22.994050364s I0312 11:40:51.506952 1 insightsuploader.go:165] Uploading latest report since 2026-03-12T11:39:16Z I0312 11:40:51.519560 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0312 11:40:51.726919 1 requests.go:87] Successfully reported id=2026-03-12T11:40:51Z x-rh-insights-request-id=c86d3dad5f7e4d0d8cc0c909f2c9cbe5, wrote=51763 I0312 11:40:51.726977 1 insightsuploader.go:187] Uploaded report successfully in 220.016177ms I0312 11:40:51.727020 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0312 11:40:51.727037 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0312 11:40:51.727048 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0312 11:40:51.733107 1 controller.go:481] The operator is healthy I0312 11:41:21.557980 1 controller.go:481] The operator is healthy I0312 11:41:57.511935 1 insightsreport.go:137] Pulling report from smart-proxy I0312 11:41:57.511968 1 insightsreport.go:149] Retrieving report I0312 11:41:57.521471 1 requests.go:110] Retrieving report for cluster: 32e98db1-fbdd-464c-8c99-1337a8a65565 I0312 11:41:57.521491 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/32e98db1-fbdd-464c-8c99-1337a8a65565/reports I0312 11:41:57.524310 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/32e98db1-fbdd-464c-8c99-1337a8a65565/reports I0312 11:41:57.721233 1 insightsreport.go:184] Report retrieved I0312 11:41:57.749412 1 insightsreport.go:239] Report retrieved correctly I0312 11:43:21.557848 1 controller.go:481] The operator is healthy I0312 11:44:21.500183 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0312 11:44:21.506143 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0312 11:44:21.506163 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0312 11:44:21.510606 1 secretconfigobserver.go:119] support secret does not exist I0312 11:45:21.558327 1 controller.go:481] The operator is healthy I0312 11:47:21.557327 1 controller.go:481] The operator is healthy I0312 11:49:21.497396 1 configmapobserver.go:84] configmaps "insights-config" not found I0312 11:49:21.502165 1 configmapobserver.go:84] configmaps "insights-config" not found I0312 11:49:21.511255 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0312 11:49:21.516570 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0312 11:49:21.516589 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0312 11:49:21.521188 1 secretconfigobserver.go:119] support secret does not exist I0312 11:49:21.556592 1 controller.go:481] The operator is healthy I0312 11:51:21.558010 1 controller.go:481] The operator is healthy I0312 11:53:21.560973 1 controller.go:481] The operator is healthy I0312 11:54:21.521948 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0312 11:54:21.532508 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0312 11:54:21.532529 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0312 11:54:21.537609 1 secretconfigobserver.go:119] support secret does not exist I0312 11:55:21.559864 1 controller.go:481] The operator is healthy I0312 11:56:19.721879 1 insightsuploader.go:146] Nothing to report since 2026-03-12T11:40:51Z I0312 11:57:21.561588 1 controller.go:481] The operator is healthy I0312 11:59:21.494998 1 configmapobserver.go:84] configmaps "insights-config" not found I0312 11:59:21.538090 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0312 11:59:21.543361 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0312 11:59:21.543380 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0312 11:59:21.547981 1 secretconfigobserver.go:119] support secret does not exist I0312 11:59:21.557209 1 controller.go:481] The operator is healthy I0312 12:01:21.557780 1 controller.go:481] The operator is healthy I0312 12:03:21.558450 1 controller.go:481] The operator is healthy I0312 12:04:21.548702 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0312 12:04:21.554441 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0312 12:04:21.554462 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0312 12:04:21.559034 1 secretconfigobserver.go:119] support secret does not exist I0312 12:05:21.561508 1 controller.go:481] The operator is healthy I0312 12:07:21.558255 1 controller.go:481] The operator is healthy I0312 12:09:21.499576 1 configmapobserver.go:84] configmaps "insights-config" not found I0312 12:09:21.504600 1 configmapobserver.go:84] configmaps "insights-config" not found I0312 12:09:21.557398 1 controller.go:481] The operator is healthy I0312 12:09:21.559212 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0312 12:09:21.564700 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0312 12:09:21.564718 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0312 12:09:21.569776 1 secretconfigobserver.go:119] support secret does not exist I0312 12:11:21.557781 1 controller.go:481] The operator is healthy I0312 12:12:38.541749 1 insightsuploader.go:146] Nothing to report since 2026-03-12T11:40:51Z I0312 12:13:21.556610 1 controller.go:481] The operator is healthy I0312 12:14:21.570003 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0312 12:14:21.576526 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0312 12:14:21.576545 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0312 12:14:21.581627 1 secretconfigobserver.go:119] support secret does not exist I0312 12:15:21.557463 1 controller.go:481] The operator is healthy