I0324 15:42:33.034904 1 cmd.go:253] Using service-serving-cert provided certificates I0324 15:42:33.035374 1 observer_polling.go:159] Starting file observer I0324 15:42:33.519904 1 operator.go:76] Starting insights-operator v0.0.0-master+$Format:%H$ I0324 15:42:33.520187 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0324 15:42:33.520619 1 secure_serving.go:57] Forcing use of http/1.1 only W0324 15:42:33.520644 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0324 15:42:33.520649 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0324 15:42:33.520654 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0324 15:42:33.520657 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0324 15:42:33.520659 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0324 15:42:33.520661 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0324 15:42:33.520794 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0324 15:42:33.524232 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"622be9c9-5151-478b-ae5e-22bdb571b27b", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "HyperShiftOnlyDynamicResourceAllocation", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLM", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "OpenShiftPodSecurityAdmission", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}} I0324 15:42:33.524224 1 operator.go:141] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNSInstall GatewayAPI GatewayAPIController HighlyAvailableArbiter HyperShiftOnlyDynamicResourceAllocation ImageStreamImportMode ImageVolume KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS ManagedBootImagesAzure ManagedBootImagesvSphere MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages PreconfiguredUDNAddresses ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SigstoreImageVerification SigstoreImageVerificationPKI StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSDualStackInstall AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureDualStackInstall AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement CBORServingAndStorage CRDCompatibilityRequirementOperator ClientsAllowCBOR ClientsPreferCBOR ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterAPIMachineManagement ClusterAPIMachineManagementVSphere ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud EtcdBackendQuota EventTTL EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall GCPDualStackInstall ImageModeStatusReporting IngressControllerDynamicConfigurationManager InsightsConfig InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesCPMS MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutableCSINodeAllocatableCount MutatingAdmissionPolicy NewOLM NewOLMBoxCutterRuntime NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterInstall NutanixMultiSubnets OSStreams OVNObservability OnPremDNSRecords OpenShiftPodSecurityAdmission ProvisioningRequestAvailable SELinuxMount ShortCertRotation SignatureStores TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeGroupSnapshot] I0324 15:42:33.525203 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0324 15:42:33.525216 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0324 15:42:33.525215 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0324 15:42:33.525232 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0324 15:42:33.525218 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0324 15:42:33.525249 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0324 15:42:33.525563 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0324 15:42:33.525887 1 secure_serving.go:213] Serving securely on [::]:8443 I0324 15:42:33.525931 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0324 15:42:33.529391 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0324 15:42:33.529420 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0324 15:42:33.529454 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0324 15:42:33.536714 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0324 15:42:33.536734 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0324 15:42:33.540941 1 secretconfigobserver.go:119] support secret does not exist I0324 15:42:33.545412 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0324 15:42:33.550217 1 secretconfigobserver.go:119] support secret does not exist I0324 15:42:33.552665 1 recorder.go:176] Pruning old reports every 5h10m43s, max age is 288h0m0s I0324 15:42:33.561932 1 periodic.go:216] Running clusterconfig gatherer I0324 15:42:33.561934 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0324 15:42:33.561952 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0324 15:42:33.561967 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0324 15:42:33.561970 1 tasks_processing.go:45] number of workers: 32 I0324 15:42:33.561971 1 insightsreport.go:296] Starting report retriever I0324 15:42:33.561976 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0324 15:42:33.561985 1 tasks_processing.go:69] worker 4 listening for tasks. I0324 15:42:33.561992 1 tasks_processing.go:69] worker 0 listening for tasks. I0324 15:42:33.561996 1 tasks_processing.go:71] worker 0 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0324 15:42:33.561998 1 tasks_processing.go:69] worker 5 listening for tasks. I0324 15:42:33.562007 1 tasks_processing.go:69] worker 1 listening for tasks. I0324 15:42:33.562009 1 tasks_processing.go:69] worker 6 listening for tasks. I0324 15:42:33.562016 1 tasks_processing.go:69] worker 7 listening for tasks. I0324 15:42:33.562014 1 tasks_processing.go:69] worker 2 listening for tasks. I0324 15:42:33.562025 1 tasks_processing.go:69] worker 20 listening for tasks. I0324 15:42:33.561992 1 tasks_processing.go:71] worker 4 working on machines task. I0324 15:42:33.562034 1 tasks_processing.go:69] worker 27 listening for tasks. I0324 15:42:33.562036 1 tasks_processing.go:69] worker 26 listening for tasks. I0324 15:42:33.562036 1 tasks_processing.go:69] worker 8 listening for tasks. I0324 15:42:33.562028 1 tasks_processing.go:69] worker 19 listening for tasks. I0324 15:42:33.562042 1 tasks_processing.go:69] worker 28 listening for tasks. I0324 15:42:33.562046 1 tasks_processing.go:69] worker 14 listening for tasks. I0324 15:42:33.562049 1 tasks_processing.go:69] worker 29 listening for tasks. I0324 15:42:33.562052 1 tasks_processing.go:69] worker 13 listening for tasks. I0324 15:42:33.562058 1 tasks_processing.go:69] worker 30 listening for tasks. I0324 15:42:33.562059 1 tasks_processing.go:69] worker 21 listening for tasks. I0324 15:42:33.562063 1 tasks_processing.go:69] worker 16 listening for tasks. I0324 15:42:33.562066 1 tasks_processing.go:69] worker 9 listening for tasks. I0324 15:42:33.562055 1 tasks_processing.go:69] worker 15 listening for tasks. I0324 15:42:33.562074 1 tasks_processing.go:69] worker 17 listening for tasks. I0324 15:42:33.562066 1 tasks_processing.go:69] worker 22 listening for tasks. I0324 15:42:33.562080 1 tasks_processing.go:69] worker 18 listening for tasks. I0324 15:42:33.562070 1 tasks_processing.go:69] worker 31 listening for tasks. I0324 15:42:33.562072 1 tasks_processing.go:69] worker 23 listening for tasks. I0324 15:42:33.562088 1 tasks_processing.go:69] worker 25 listening for tasks. I0324 15:42:33.562081 1 tasks_processing.go:69] worker 24 listening for tasks. I0324 15:42:33.562081 1 tasks_processing.go:69] worker 10 listening for tasks. I0324 15:42:33.562089 1 tasks_processing.go:69] worker 12 listening for tasks. I0324 15:42:33.562089 1 tasks_processing.go:69] worker 11 listening for tasks. I0324 15:42:33.562020 1 tasks_processing.go:69] worker 3 listening for tasks. I0324 15:42:33.562167 1 tasks_processing.go:71] worker 11 working on service_accounts task. I0324 15:42:33.562172 1 tasks_processing.go:71] worker 22 working on sap_pods task. I0324 15:42:33.562177 1 tasks_processing.go:71] worker 5 working on openstack_controlplanes task. I0324 15:42:33.562188 1 tasks_processing.go:71] worker 27 working on storage_cluster task. I0324 15:42:33.562220 1 tasks_processing.go:71] worker 16 working on tsdb_status task. I0324 15:42:33.562278 1 tasks_processing.go:71] worker 15 working on dvo_metrics task. I0324 15:42:33.562317 1 tasks_processing.go:71] worker 7 working on networks task. I0324 15:42:33.562372 1 tasks_processing.go:71] worker 1 working on silenced_alerts task. I0324 15:42:33.562411 1 tasks_processing.go:71] worker 2 working on openshift_machine_api_events task. I0324 15:42:33.562466 1 tasks_processing.go:71] worker 20 working on schedulers task. I0324 15:42:33.562506 1 tasks_processing.go:71] worker 26 working on image task. I0324 15:42:33.562565 1 tasks_processing.go:71] worker 21 working on crds task. I0324 15:42:33.562443 1 tasks_processing.go:71] worker 9 working on proxies task. I0324 15:42:33.562707 1 tasks_processing.go:71] worker 12 working on install_plans task. I0324 15:42:33.562292 1 tasks_processing.go:71] worker 25 working on jaegers task. I0324 15:42:33.562172 1 tasks_processing.go:71] worker 13 working on validating_webhook_configurations task. I0324 15:42:33.562584 1 tasks_processing.go:71] worker 8 working on openstack_dataplanedeployments task. I0324 15:42:33.562167 1 tasks_processing.go:71] worker 3 working on ceph_cluster task. I0324 15:42:33.562306 1 tasks_processing.go:71] worker 23 working on cost_management_metrics_configs task. I0324 15:42:33.562311 1 tasks_processing.go:71] worker 30 working on lokistack task. I0324 15:42:33.562453 1 tasks_processing.go:71] worker 6 working on container_images task. I0324 15:42:33.562460 1 tasks_processing.go:71] worker 17 working on pod_network_connectivity_checks task. I0324 15:42:33.562589 1 tasks_processing.go:71] worker 19 working on sap_datahubs task. I0324 15:42:33.562594 1 tasks_processing.go:71] worker 28 working on container_runtime_configs task. I0324 15:42:33.562597 1 tasks_processing.go:71] worker 14 working on node_features task. I0324 15:42:33.562602 1 tasks_processing.go:71] worker 29 working on node_logs task. I0324 15:42:33.562608 1 tasks_processing.go:71] worker 10 working on machine_sets task. I0324 15:42:33.562612 1 tasks_processing.go:71] worker 24 working on aggregated_monitoring_cr_names task. I0324 15:42:33.562302 1 tasks_processing.go:71] worker 31 working on machine_healthchecks task. I0324 15:42:33.562297 1 tasks_processing.go:71] worker 18 working on image_pruners task. I0324 15:42:33.565135 1 tasks_processing.go:71] worker 5 working on infrastructures task. I0324 15:42:33.565144 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 2.947911ms to process 0 records I0324 15:42:33.565353 1 tasks_processing.go:71] worker 4 working on sap_config task. E0324 15:42:33.565364 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0324 15:42:33.565493 1 gather.go:177] gatherer "clusterconfig" function "machines" took 3.30924ms to process 0 records I0324 15:42:33.565821 1 tasks_processing.go:71] worker 22 working on machine_configs task. I0324 15:42:33.565836 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 3.641071ms to process 0 records E0324 15:42:33.567035 1 gather_silenced_alerts.go:52] Unable to retrieve silenced alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp 172.30.19.227:9094: connect: connection refused I0324 15:42:33.567054 1 tasks_processing.go:71] worker 1 working on operators task. E0324 15:42:33.567050 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.208.16:9091: connect: connection refused E0324 15:42:33.567065 1 gather.go:140] gatherer "clusterconfig" function "silenced_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp 172.30.19.227:9094: connect: connection refused I0324 15:42:33.567081 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 4.665961ms to process 0 records E0324 15:42:33.567093 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.208.16:9091: connect: connection refused I0324 15:42:33.567101 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 4.831506ms to process 0 records I0324 15:42:33.567221 1 tasks_processing.go:71] worker 16 working on clusterroles task. I0324 15:42:33.569123 1 tasks_processing.go:71] worker 27 working on overlapping_namespace_uids task. I0324 15:42:33.569162 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 6.920222ms to process 0 records I0324 15:42:33.569195 1 tasks_processing.go:71] worker 3 working on cluster_apiserver task. I0324 15:42:33.569215 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 6.107655ms to process 0 records I0324 15:42:33.569252 1 tasks_processing.go:71] worker 23 working on ingress_certificates task. I0324 15:42:33.569284 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 6.036342ms to process 0 records I0324 15:42:33.573971 1 tasks_processing.go:71] worker 30 working on support_secret task. I0324 15:42:33.573999 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 10.677131ms to process 0 records E0324 15:42:33.574010 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0324 15:42:33.574016 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 10.527687ms to process 0 records E0324 15:42:33.574046 1 gather_node_features.go:86] GatherNodeFeatures: NodeFeatures resource not found in openshift-nfd namespace (may not be installed) I0324 15:42:33.574055 1 gather.go:177] gatherer "clusterconfig" function "node_features" took 10.363404ms to process 0 records I0324 15:42:33.574083 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 11.250588ms to process 0 records I0324 15:42:33.574131 1 tasks_processing.go:71] worker 25 working on nodenetworkstates task. I0324 15:42:33.574221 1 tasks_processing.go:71] worker 17 working on config_maps task. I0324 15:42:33.574413 1 tasks_processing.go:71] worker 20 working on olm_operators task. I0324 15:42:33.574279 1 tasks_processing.go:71] worker 14 working on machine_config_pools task. I0324 15:42:33.574610 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=a63d7f62468eeef823474a81ff82c49ab3e9cb23ca9c63129dabb568eae90b95 I0324 15:42:33.574650 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 11.905991ms to process 1 records I0324 15:42:33.574742 1 tasks_processing.go:71] worker 7 working on mutating_webhook_configurations task. I0324 15:42:33.574922 1 recorder.go:75] Recording config/network with fingerprint=88926a4473bebed3da5d74b81e5c18aa77692eb42a193ca2e6ba018e4fb6653d I0324 15:42:33.574939 1 gather.go:177] gatherer "clusterconfig" function "networks" took 12.150957ms to process 1 records I0324 15:42:33.575008 1 recorder.go:75] Recording config/proxy with fingerprint=aec13ff8b81612cf386cf2f8e6377e22d6a3c811d4b06e61719c59e0f0ea3933 I0324 15:42:33.575016 1 tasks_processing.go:71] worker 9 working on image_registries task. I0324 15:42:33.575021 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 11.944269ms to process 1 records I0324 15:42:33.575032 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 9.307008ms to process 0 records I0324 15:42:33.575046 1 tasks_processing.go:71] worker 4 working on openshift_logging task. I0324 15:42:33.575942 1 tasks_processing.go:71] worker 31 working on openstack_version task. E0324 15:42:33.575976 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0324 15:42:33.575992 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 11.779396ms to process 0 records I0324 15:42:33.576104 1 tasks_processing.go:71] worker 26 working on openstack_dataplanenodesets task. I0324 15:42:33.576218 1 recorder.go:75] Recording config/image with fingerprint=da673319477fd155f3570693064f4dc13e51a61017d181dec1493f06ee41d29f I0324 15:42:33.576235 1 gather.go:177] gatherer "clusterconfig" function "image" took 13.548357ms to process 1 records I0324 15:42:33.576623 1 tasks_processing.go:71] worker 28 working on pdbs task. I0324 15:42:33.576636 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 12.950436ms to process 0 records W0324 15:42:33.577803 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0324 15:42:33.582341 1 tasks_processing.go:71] worker 2 working on active_alerts task. I0324 15:42:33.582371 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 19.864871ms to process 0 records I0324 15:42:33.582689 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 18.33747ms to process 0 records I0324 15:42:33.582724 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 19.398202ms to process 0 records I0324 15:42:33.582761 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 18.870232ms to process 0 records I0324 15:42:33.582743 1 tasks_processing.go:71] worker 10 working on ingress task. I0324 15:42:33.582744 1 controller.go:129] Initializing last reported time to 0001-01-01T00:00:00Z I0324 15:42:33.583032 1 controller.go:368] The initial operator extension status is healthy I0324 15:42:33.582779 1 tasks_processing.go:71] worker 8 working on version task. I0324 15:42:33.583146 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0324 15:42:33.583600 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0324 15:42:33.583632 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0324 15:42:33.582996 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=e91163194c1d1b2fc5faee45a78d9499dad48a9807bff020594527c3be39d759 I0324 15:42:33.584170 1 controller.go:531] The operator is still being initialized I0324 15:42:33.584188 1 controller.go:554] The operator is healthy I0324 15:42:33.584383 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=3411afd78d566ffd094fef3aa990187bcd0434d5f9ba92d64790e937945c1b5c I0324 15:42:33.584599 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=a0cf463d17610e55be042f484de4351f8c4f4f3947f441a586e7438c2f3325c6 I0324 15:42:33.584632 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s I0324 15:42:33.584709 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=4913bcb52a94067bab080659091fe2c3c4dde2898e99518bf85c6bedeff76ad8 I0324 15:42:33.584791 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=0d27848c37e0dfebce0e448dbb43ade4d03d22b2bcbad005c53d9521a5e22ef7 I0324 15:42:33.584886 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=f8b4bbaa067b8ceef3c0974ebf74be53256018bd0b3b5921443e5faf356ec766 I0324 15:42:33.584989 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=e0eba5319b4657a69168c4ab70777e87ff155e3e355ed6eb019c430f42564a89 I0324 15:42:33.585078 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0324 15:42:33.585084 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=95c98c3a6f20e49a43cf0e5aa6f27215c0174470d581f7f1d78ebb7c7af0b5fd I0324 15:42:33.585302 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=f2dd97106690654245975a83b5e07d49367c96ccca2c7a7f1c6002d5211c79bd W0324 15:42:33.585414 1 operator.go:328] started I0324 15:42:33.585454 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0324 15:42:33.585542 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=7c3c0553f387f2f3337a4ed03dd20f7bb9c99a5f067e1d68e43d04a5979886a0 I0324 15:42:33.585639 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=1afaee51527cce8b16e2e2f11d78b2ddaa803b6f573d32cae6a1e09ba2aa837b I0324 15:42:33.585715 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=eeb7e1945c4fe26e65d889f1c9e74e70252f8854c8e641a892edcce4fe7b26af I0324 15:42:33.585784 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=ff0884619f4fcc15306ba5740cd2ebdfe0a9a8307afd983dbcb365a1c75d4c40 I0324 15:42:33.585816 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 19.744501ms to process 13 records E0324 15:42:33.585845 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0324 15:42:33.585902 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0324 15:42:33.583009 1 tasks_processing.go:71] worker 19 working on certificate_signing_requests task. I0324 15:42:33.586021 1 tasks_processing.go:71] worker 22 working on qemu_kubevirt_launcher_logs task. I0324 15:42:33.583004 1 tasks_processing.go:71] worker 13 working on oauths task. I0324 15:42:33.585950 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 16.905001ms to process 1 records I0324 15:42:33.586809 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 19.296291ms to process 0 records I0324 15:42:33.586820 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 9.579028ms to process 0 records I0324 15:42:33.586827 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 9.420008ms to process 0 records I0324 15:42:33.586840 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 8.960334ms to process 0 records I0324 15:42:33.586900 1 tasks_processing.go:71] worker 25 working on feature_gates task. I0324 15:42:33.586998 1 tasks_processing.go:71] worker 29 working on operators_pods_and_events task. I0324 15:42:33.586974 1 tasks_processing.go:71] worker 4 working on authentication task. I0324 15:42:33.587002 1 tasks_processing.go:71] worker 3 working on nodenetworkconfigurationpolicies task. I0324 15:42:33.587011 1 tasks_processing.go:71] worker 14 working on machine_autoscalers task. E0324 15:42:33.587349 1 gather_active_alerts.go:64] Unable to retrieve most recent alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp 172.30.19.227:9094: connect: connection refused I0324 15:42:33.587693 1 recorder.go:75] Recording config/apiserver with fingerprint=046563cc8809a47768953a6fd3169d9ce269a91c99f3937d2dabf27676a36995 I0324 15:42:33.587714 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 14.824053ms to process 1 records I0324 15:42:33.587823 1 tasks_processing.go:71] worker 18 working on nodes task. I0324 15:42:33.587966 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=1e02ffb5f734afa712b492c9d570c159a1276969fd86a96d5b8389474e167eac I0324 15:42:33.587981 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 19.873251ms to process 1 records I0324 15:42:33.588179 1 tasks_processing.go:71] worker 5 working on monitoring_persistent_volumes task. I0324 15:42:33.588735 1 recorder.go:75] Recording config/infrastructure with fingerprint=812056ee84f6352420f9e927e6aaf720d9e47b265a9c29a6944df9053e2782e2 I0324 15:42:33.588754 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 19.455762ms to process 1 records E0324 15:42:33.588763 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0324 15:42:33.588770 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 13.032965ms to process 0 records E0324 15:42:33.588777 1 gather.go:140] gatherer "clusterconfig" function "active_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp 172.30.19.227:9094: connect: connection refused I0324 15:42:33.588790 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 4.979499ms to process 0 records I0324 15:42:33.588808 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0324 15:42:33.588819 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 18.908246ms to process 1 records I0324 15:42:33.588829 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 12.357036ms to process 0 records I0324 15:42:33.588837 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 12.506348ms to process 0 records I0324 15:42:33.588841 1 tasks_processing.go:71] worker 2 working on metrics task. I0324 15:42:33.588999 1 tasks_processing.go:74] worker 27 stopped. I0324 15:42:33.589012 1 tasks_processing.go:71] worker 26 working on storage_classes task. I0324 15:42:33.589126 1 tasks_processing.go:74] worker 30 stopped. I0324 15:42:33.589155 1 tasks_processing.go:74] worker 31 stopped. I0324 15:42:33.589757 1 tasks_processing.go:74] worker 9 stopped. I0324 15:42:33.590191 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=f0f611c5102850d6a5566f60fbe9ff30319bf208f989b06493eae50f3e3b1ed8 I0324 15:42:33.590211 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 14.725534ms to process 1 records I0324 15:42:33.590299 1 tasks_processing.go:74] worker 7 stopped. I0324 15:42:33.590371 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=ecd0f04e5700f8dd43294ba60ac8b3c374755c5dd686d280a59b6075f6ba45b6 I0324 15:42:33.590419 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=940746033d9eefb2c9e70bd77725060c63b8759eb89a98050380743b2ab426a3 I0324 15:42:33.590464 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=2163ca733bc1999c37e32ce7d7e938f27848fb30e44c209cf36c19f38972366d I0324 15:42:33.590474 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 15.071912ms to process 3 records E0324 15:42:33.591070 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.208.16:9091: connect: connection refused I0324 15:42:33.591103 1 tasks_processing.go:74] worker 2 stopped. E0324 15:42:33.591124 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.208.16:9091: connect: connection refused I0324 15:42:33.591137 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 2.248383ms to process 0 records I0324 15:42:33.604028 1 tasks_processing.go:74] worker 14 stopped. I0324 15:42:33.604047 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 16.775819ms to process 0 records I0324 15:42:33.604255 1 gather_logs.go:145] no pods in namespace were found I0324 15:42:33.604394 1 tasks_processing.go:74] worker 21 stopped. I0324 15:42:33.605274 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0324 15:42:33.605332 1 controller.go:254] Source scaController *sca.Controller is not ready I0324 15:42:33.605358 1 controller.go:254] Source clusterTransferController *clustertransfer.Controller is not ready I0324 15:42:33.605382 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0324 15:42:33.605404 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0324 15:42:33.605437 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=5a5a1c9776f7b2574fdbe8077b59d7766592de2a26f0686daf5f4823975d54b7 I0324 15:42:33.605460 1 controller.go:531] The operator is still being initialized I0324 15:42:33.605663 1 controller.go:554] The operator is healthy I0324 15:42:33.605868 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=e1194529aa665f394d8d4f177a2be29560bd6dcb51e1be2b8fe78176f3ef2184 I0324 15:42:33.605945 1 gather.go:177] gatherer "clusterconfig" function "crds" took 41.41034ms to process 2 records I0324 15:42:33.606234 1 recorder.go:75] Recording config/ingress with fingerprint=83e2e71055e5e800c32e6a44484c7a3f9737cadedaeb218dfa065f2fbb97c289 I0324 15:42:33.606335 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 21.425885ms to process 1 records I0324 15:42:33.606416 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 18.263851ms to process 0 records I0324 15:42:33.606453 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 17.260091ms to process 0 records I0324 15:42:33.606628 1 tasks_processing.go:74] worker 22 stopped. I0324 15:42:33.606296 1 tasks_processing.go:74] worker 10 stopped. I0324 15:42:33.606680 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=ac890b95ba8225779eacb9acc7f7f82acd89b12089f0e4eb53c77bc5537f9a58 I0324 15:42:33.606770 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=b205a377f233ec564e8be51b623f57ed14d680a0c8540c83192827eb7ae6c3e1 I0324 15:42:33.606827 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=6c32422362ffd2588bd8b6883648bfdfbd97de1637fb2d8e877cab6cd75792a4 I0324 15:42:33.607053 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=1b0fc714a121e41a98331f07d660394280674dcd196e94e0950769cdb86b5925 I0324 15:42:33.607146 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=21d5097f43ce8857616109bf56d5f7db556f953156a1da20a791660e4851dd3d I0324 15:42:33.607185 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=0ffd99a839dfd54b7fec91239233ded39c1970247c9cfd85bec71ebabea40b8d I0324 15:42:33.607212 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=74bc31bae0a789c210e96c7183d768cb37d438e9fa0a9c026694a11dc698740d I0324 15:42:33.607245 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=db68eb7c71dd8d7737e37abbc314d0cf04b85057749483b8365ce2975d0b73be I0324 15:42:33.607297 1 tasks_processing.go:74] worker 28 stopped. I0324 15:42:33.607297 1 tasks_processing.go:74] worker 3 stopped. I0324 15:42:33.607312 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=912bbff710be978c3dfc4ed67dd972d8e8facf789db131782a6687fda887a16e I0324 15:42:33.607335 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=df992dd69490cc7bf86be543621fc9b1f5dc6ebf53f520a5faddb3cc4ba2d040 I0324 15:42:33.607346 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 29.087125ms to process 10 records I0324 15:42:33.607711 1 tasks_processing.go:74] worker 18 stopped. I0324 15:42:33.608706 1 recorder.go:75] Recording config/node/ip-10-0-0-64.ec2.internal with fingerprint=977e6e437eacf373c31c0bad33479d35a33a414d2f59e207a42d953e798f0518 I0324 15:42:33.608834 1 recorder.go:75] Recording config/node/ip-10-0-1-113.ec2.internal with fingerprint=c23d10fa4006e689e7bd5e0f7a9394158d69b3bf95f339bb2d337d0ee46016aa I0324 15:42:33.608915 1 recorder.go:75] Recording config/node/ip-10-0-2-142.ec2.internal with fingerprint=25413a1e909f376c9f984352e49af116b9d366578a909c02ff68473ae8bc251f I0324 15:42:33.608928 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 19.872401ms to process 3 records I0324 15:42:33.609007 1 tasks_processing.go:74] worker 13 stopped. I0324 15:42:33.609211 1 recorder.go:75] Recording config/oauth with fingerprint=132ad6197da137005d464fb44ef43c4c5acb4f5e731c4f174ff4f9770aa4c6c2 I0324 15:42:33.609254 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 21.235027ms to process 1 records I0324 15:42:33.609387 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=3d275fb50d5e896cf56ec5eeadf4b5d1d478eb07888aa97ae714a84545efa054 I0324 15:42:33.609439 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=1023e55f7bb169ad4a215ccbaf97fc3655f821ab486c5126e3be2ff3fc7404c9 I0324 15:42:33.609471 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 19.529871ms to process 2 records I0324 15:42:33.609501 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 45.295015ms to process 0 records I0324 15:42:33.609524 1 tasks_processing.go:74] worker 24 stopped. I0324 15:42:33.609399 1 tasks_processing.go:74] worker 26 stopped. I0324 15:42:33.609532 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 21.252885ms to process 0 records I0324 15:42:33.609540 1 tasks_processing.go:74] worker 5 stopped. I0324 15:42:33.609821 1 tasks_processing.go:74] worker 25 stopped. I0324 15:42:33.609888 1 recorder.go:75] Recording config/featuregate with fingerprint=46017bbf7d2ab8e863c9f56e74bd9e2de6162e00933372923e91ad9fd9b277a8 I0324 15:42:33.609929 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 22.581252ms to process 1 records I0324 15:42:33.610281 1 tasks_processing.go:74] worker 20 stopped. I0324 15:42:33.610420 1 recorder.go:75] Recording config/olm_operators with fingerprint=57a6095aa7a16f75eded9683c0baceffe7ac6ee1ba8533d941ba1be564d58f03 I0324 15:42:33.610463 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 35.829765ms to process 1 records I0324 15:42:33.610637 1 tasks_processing.go:74] worker 16 stopped. I0324 15:42:33.610777 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=f9d53753c0024d4fb1e7e135e3c5b2b313268fe51f42b801dcdf78d706d73220 I0324 15:42:33.610979 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=33d5d602bc58f4f227f1ed35be8e962c0f8e0d6ff53587126e146bdfcee2b396 I0324 15:42:33.610996 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 43.188278ms to process 2 records I0324 15:42:33.611050 1 tasks_processing.go:74] worker 4 stopped. I0324 15:42:33.611358 1 recorder.go:75] Recording config/authentication with fingerprint=7e1b2fd06d50926f17e315201e3359273da4996182510f0263a6977a04148da2 I0324 15:42:33.611376 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 23.934851ms to process 1 records I0324 15:42:33.612661 1 tasks_processing.go:74] worker 19 stopped. I0324 15:42:33.612675 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 26.667957ms to process 0 records I0324 15:42:33.612908 1 tasks_processing.go:74] worker 29 stopped. I0324 15:42:33.612922 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 25.859111ms to process 0 records I0324 15:42:33.614319 1 tasks_processing.go:74] worker 8 stopped. I0324 15:42:33.614700 1 recorder.go:75] Recording config/version with fingerprint=a4630810cadcaf75c6177c387408a7968c4b4547d92e1627ac77d9f6bdfd3937 I0324 15:42:33.614719 1 recorder.go:75] Recording config/id with fingerprint=d7f872b59fb1000907436f4f87d5d5ac17ae25a9855c18d88c77187dc8b77456 I0324 15:42:33.614741 1 gather.go:177] gatherer "clusterconfig" function "version" took 31.240856ms to process 2 records I0324 15:42:33.622952 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0324 15:42:33.625491 1 tasks_processing.go:74] worker 6 stopped. I0324 15:42:33.625508 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0324 15:42:33.625514 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0324 15:42:33.625584 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0324 15:42:33.627349 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-5b5d9b4544-27msz with fingerprint=d183d64ede4c012a690a3030038ca537a47af65cf72025004350e5cbed1f000e I0324 15:42:33.627725 1 recorder.go:75] Recording config/pod/openshift-ovn-kubernetes/ovnkube-node-2gvxt with fingerprint=c791144e5c38fa730d528929967eadcb8a83da82ff4caa44545e7167cd391954 I0324 15:42:33.628365 1 recorder.go:75] Recording config/pod/openshift-ovn-kubernetes/ovnkube-node-w28k8 with fingerprint=f6f6eafab5e2b6e92e490b3f70944b5d914b2b45ce70777c2221fe34a79fa342 I0324 15:42:33.628642 1 recorder.go:75] Recording config/running_containers with fingerprint=8cd794220e98beff6dc1fe4f57bc90052ccd1b7fa85143003775ec8dfed553cf I0324 15:42:33.628692 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 62.148016ms to process 4 records I0324 15:42:33.628724 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 65.036012ms to process 0 records I0324 15:42:33.628751 1 tasks_processing.go:74] worker 0 stopped. I0324 15:42:33.629944 1 tasks_processing.go:74] worker 17 stopped. E0324 15:42:33.629966 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0324 15:42:33.629975 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0324 15:42:33.629981 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0324 15:42:33.629993 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=c8a9805405aae7d04b7f03c09c51097942070877009b6daebd5573ef8127446e I0324 15:42:33.630009 1 base_controller.go:82] Caches are synced for ConfigController I0324 15:42:33.630028 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0324 15:42:33.630133 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0324 15:42:33.630180 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0324 15:42:33.630187 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=3b70b9787487b548204d806ebebf90b0c2d460145e34cb33c0d8d3a7a011d907 I0324 15:42:33.630205 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0324 15:42:33.630292 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0324 15:42:33.630533 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0324 15:42:33.630542 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 55.631471ms to process 7 records E0324 15:42:33.630554 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0324 15:42:33.630559 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2p85tm5tkg18fgnuc739urkgf069jgs6-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2p85tm5tkg18fgnuc739urkgf069jgs6-primary-cert-bundle-secret" not found I0324 15:42:33.630605 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=7404f3d53b361272daec4c2749668918318553a23d6c57cf1bffb00cb0e7310a I0324 15:42:33.630614 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 60.885119ms to process 1 records I0324 15:42:33.630620 1 tasks_processing.go:74] worker 23 stopped. I0324 15:42:33.650801 1 configmapobserver.go:84] configmaps "insights-config" not found I0324 15:42:33.677127 1 cluster_transfer.go:102] no available accepted cluster transfer I0324 15:42:33.677141 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0324 15:42:33.685939 1 base_controller.go:82] Caches are synced for LoggingSyncer I0324 15:42:33.685951 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0324 15:42:33.781778 1 sca.go:266] etc-pki-entitlement secret successfully created I0324 15:42:33.781794 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated W0324 15:42:34.578418 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0324 15:42:35.577751 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0324 15:42:36.028086 1 tasks_processing.go:74] worker 1 stopped. I0324 15:42:36.028195 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=d0ea474b1dad4052a10ea7fdaaf188b3f62054c41a4791495a0b7b282c473deb I0324 15:42:36.028242 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0324 15:42:36.028294 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=24129ba0ec0d1a5da9d94a8927b317bf19b601eceb484abebfbc81556de32c97 I0324 15:42:36.028306 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0324 15:42:36.028325 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=a50604c27ccac1a1be5ee9883f3623c6ebe7d8e1d2b74974ab60adfac9480899 I0324 15:42:36.028341 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0324 15:42:36.028363 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=68248200cc86cc780dfad5d8786ac9a119fe9560fb80e9c66a4f7e40fb6799ec I0324 15:42:36.028387 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=3d26572a5b03331f90624d95cac4293d2176fa73132511f485685064f13a2465 I0324 15:42:36.028409 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=b798339841191e8406c0282bee806206be89c9a31818c29a88f4d05eff108807 I0324 15:42:36.028438 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=a7982ef9d33116a9a5d53a840c2ea3f0ba7e18367e59942ad19e8139df9bdcf1 I0324 15:42:36.028707 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=8937191c71883560f53427725e449b2c1405451c781b3a8f753b4448bf08c1ce I0324 15:42:36.028725 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=60aaf59a92ad8a1bb7aa5d6cb9df3fbdf75c2b038c023af2525bf4449389c55f I0324 15:42:36.028733 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0324 15:42:36.028748 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=8718e0848c021e51718c95fe3c62deae6c6ba4a942f45c5a9f6ec0665ea95a87 I0324 15:42:36.028758 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0324 15:42:36.028773 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=a384c7928c70c8a2c97a7750bf00607388fc29587d0e092c225b023a8f00a1fd I0324 15:42:36.028782 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0324 15:42:36.028799 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=d0f52ea0e756f0147c91ccdb67cb6fe6bfe4578ff796b9c27170dbfa80108b92 I0324 15:42:36.028809 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0324 15:42:36.028828 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=3617a34b5dda106ab666a1325b65c8eef0af9a12c299a7d8618b52862b95e767 I0324 15:42:36.028929 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=77eaa052124f432ff5492126cfc32f2606a3ddc93f7cd3d586837ba24cf7bd70 I0324 15:42:36.028937 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0324 15:42:36.028945 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0324 15:42:36.028974 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0324 15:42:36.028995 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=f89708e80643803fac0db5096100d6bcc028b3ca70d9598a6a529c5b9b3a93f5 I0324 15:42:36.029019 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=db40b21b57f0c85f447ad1e11fce536517f66bf8891ddba011cce520eb1645f9 I0324 15:42:36.029027 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0324 15:42:36.029043 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=ea3b8db96817074aaa6ad0587ee65bfb6bf5062a6bb5789c2b53ecfbc00e64a9 I0324 15:42:36.029052 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0324 15:42:36.029069 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=3b5262c9638964b54a4b892e2e40700ca61cf63238bc47d8967ab0ca96a60408 I0324 15:42:36.029077 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0324 15:42:36.029089 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=95acb64395e2a941965278c5eb7d5509e4df5c777e8a591ada58eb62499abb4c I0324 15:42:36.029104 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=3fd1d7af40ae26e59b9cbc61c667c995c1d06d08c3e84f26e9f65ff5a17a926f I0324 15:42:36.029119 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=06fce4c60d2e12084f1fe8f67149c2c59ed043b6d34a606e74187f0314ab1d3f I0324 15:42:36.029140 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=5ba01375895829e86ea70fe00fbe408c359c9e670e501e1041768857f6bc9d69 I0324 15:42:36.029149 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0324 15:42:36.029176 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=c4263c68300eeff4e4de58cf6ea1070fec3c2ece049f138a9b573764dccee4f3 I0324 15:42:36.029194 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0324 15:42:36.029203 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0324 15:42:36.029210 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.46101681s to process 39 records W0324 15:42:36.577851 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0324 15:42:37.578314 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0324 15:42:38.578527 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0324 15:42:38.578569 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0324 15:42:38.578588 1 tasks_processing.go:74] worker 15 stopped. E0324 15:42:38.578599 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0324 15:42:38.578609 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0324 15:42:38.578619 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.016261552s to process 1 records I0324 15:42:46.383246 1 tasks_processing.go:74] worker 12 stopped. I0324 15:42:46.383315 1 recorder.go:75] Recording config/installplans with fingerprint=f17dbfacc3bfddf27ca3b213b39495434cd4c4e9e3dbd69566ffb3845bbcf539 I0324 15:42:46.383330 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.820478069s to process 1 records I0324 15:42:47.179553 1 tasks_processing.go:74] worker 11 stopped. I0324 15:42:47.179836 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=16e4c55b4d63cefcfc7800aa646e617d46f0ec5f940c3fd1a629fc5ae985c964 I0324 15:42:47.179855 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.617367748s to process 1 records E0324 15:42:47.179920 1 periodic.go:254] "Unhandled Error" err="clusterconfig failed after 13.617s with: function \"machines\" failed with an error, function \"silenced_alerts\" failed with an error, function \"tsdb_status\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"machine_configs\" failed with an error, function \"support_secret\" failed with an error, function \"active_alerts\" failed with an error, function \"metrics\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0324 15:42:47.181022 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "machines" failed with an error, function "silenced_alerts" failed with an error, function "tsdb_status" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machine_healthchecks" failed with an error, function "machine_configs" failed with an error, function "support_secret" failed with an error, function "active_alerts" failed with an error, function "metrics" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0324 15:42:47.181037 1 periodic.go:216] Running workloads gatherer I0324 15:42:47.181052 1 tasks_processing.go:45] number of workers: 2 I0324 15:42:47.181061 1 tasks_processing.go:69] worker 1 listening for tasks. I0324 15:42:47.181066 1 tasks_processing.go:71] worker 1 working on workload_info task. I0324 15:42:47.181074 1 tasks_processing.go:69] worker 0 listening for tasks. I0324 15:42:47.181101 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0324 15:42:47.195387 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-142.ec2.internal... I0324 15:42:47.195396 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-64.ec2.internal... I0324 15:42:47.195411 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-113.ec2.internal... I0324 15:42:47.246815 1 tasks_processing.go:74] worker 0 stopped. I0324 15:42:47.246837 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 65.691221ms to process 0 records I0324 15:42:52.873725 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 6s I0324 15:42:52.897874 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 25s for image data I0324 15:42:52.908857 1 gather_workloads_info.go:387] No image sha256:1aabd00f813e1afccdab16a845e6346288f5cf7e320fa5bb97c683702191c575 (12ms) I0324 15:42:52.918184 1 gather_workloads_info.go:387] No image sha256:bbb5af28bbed69e8189825fdc6665f4e8dbcd0c0c83fa479943ea102fe9452d6 (9ms) I0324 15:42:52.926002 1 gather_workloads_info.go:387] No image sha256:5efa227ddac82e56eddc03761e220af1f2d6a4b22ee00b2baf426523f26c0319 (8ms) I0324 15:42:52.933232 1 gather_workloads_info.go:387] No image sha256:3e889ea85c9c4748c548b84ae135a8adc88f42026ce8073538ca2fe7d89b8004 (7ms) I0324 15:42:52.940584 1 gather_workloads_info.go:387] No image sha256:9dee9b6866db30456c277ff1f7a114dd4ce25bff8aced70218623af9423ec3cf (7ms) I0324 15:42:52.947778 1 gather_workloads_info.go:387] No image sha256:50b7c5185c4a6e63145753415c92723c59136d14b951ea0f4f06739b00893329 (7ms) I0324 15:42:52.954734 1 gather_workloads_info.go:387] No image sha256:1c325fa866a794c56a7b30ca9d4fe225606566651bb12a2ecce8f60cbddb2809 (7ms) I0324 15:42:52.961686 1 gather_workloads_info.go:387] No image sha256:3e772c41ef85aaccf63c409ee72b43d632b36d4cc66a3790835dffdcf9b7433e (7ms) I0324 15:42:52.969357 1 gather_workloads_info.go:387] No image sha256:60648d27017709a420e43353f9a83d7c466017b3b771200672a4dbc3c3f7a3ae (8ms) I0324 15:42:52.976823 1 gather_workloads_info.go:387] No image sha256:188e4e2d697e2ca3c7c16772c0ab12253f00bd8433d8691ed7058fe7bafbbbf4 (7ms) I0324 15:42:53.006092 1 gather_workloads_info.go:387] No image sha256:6521242b8805e70936f8e49c7d50a319adaeb73b4e5dec4693b0f93a34e397c5 (29ms) I0324 15:42:53.107816 1 gather_workloads_info.go:387] No image sha256:03a7d45ceb838c746bdcdac1a5cc02d26273b319d9da4498bb3ee31c3e130b43 (102ms) I0324 15:42:53.206414 1 gather_workloads_info.go:398] Found image sha256:db40a3d4e50016c645fdb4a1f389dfcdbc9febc10c174816d681abe5c14153cd (99ms) I0324 15:42:53.306379 1 gather_workloads_info.go:387] No image sha256:862d5befeba504d5058264cf65f16fc307eea4eb3914fbbbf40fbe61714b0958 (100ms) I0324 15:42:53.406574 1 gather_workloads_info.go:387] No image sha256:25d9148ea34149935323231c64952679727b6f7f235febc3be1df63e0bf18c5e (100ms) I0324 15:42:53.505900 1 gather_workloads_info.go:387] No image sha256:30a054cfe1b8330bd57d762921ca5cd281ad2fdd7994fd3093ffadb5d767a564 (99ms) I0324 15:42:53.605563 1 gather_workloads_info.go:387] No image sha256:79fca57a4fd5ef00472a287218b84061b941905c61fcc55e89073dd93fa77648 (100ms) I0324 15:42:53.706159 1 gather_workloads_info.go:387] No image sha256:0b2cc6aadb1abc1d35a55da0c9884eb9426d7ff7a97174daeb09bb1e1a23cfae (101ms) I0324 15:42:53.807503 1 gather_workloads_info.go:387] No image sha256:7f426776d3ef70b585018b8d813f5eedd70373536681c6240df224ebf9ff6329 (101ms) I0324 15:42:53.905739 1 gather_workloads_info.go:387] No image sha256:3d68a9523f4c5c22a48dfa151bbfde78b2f360697afe3b9d8bfb53e8c359bf95 (98ms) I0324 15:42:54.006045 1 gather_workloads_info.go:387] No image sha256:876fb57ffb255ed75a452405bcd1d116788a34b19264886e6e087ff544c17083 (100ms) I0324 15:42:54.106492 1 gather_workloads_info.go:387] No image sha256:7dd9584d291967387f2a1ade991ad3bf8565bb59f49b013217951399e8273498 (100ms) I0324 15:42:54.208525 1 gather_workloads_info.go:387] No image sha256:11de6fe1741f5dd485e59ee2e703868badda81de0a179513be70b6ac9fc4655f (102ms) I0324 15:42:54.306957 1 gather_workloads_info.go:387] No image sha256:12e6985890d3c549f7cc9881892793b27277f72e2739cb910aaf9f4dd7215df1 (98ms) I0324 15:42:54.405619 1 gather_workloads_info.go:387] No image sha256:72025568de4c6786f56594714efbe3c5db3c2791fb1a7a47e6361e5b53a35107 (99ms) I0324 15:42:54.507165 1 gather_workloads_info.go:387] No image sha256:17e68da37c74bdc95a4eb43b51ce671027807ee610a4ee5237324a8c0dc9d044 (102ms) I0324 15:42:54.608227 1 gather_workloads_info.go:387] No image sha256:3666bcb850e1518aec86d322af52e95f725602ad0ab770d2b1e5b39b6dbde0b5 (101ms) I0324 15:42:54.706236 1 gather_workloads_info.go:387] No image sha256:0c90ca6acbe2f4fdbd253ae22f5b3286247c6940321f7d2b9801aea21ef5269f (98ms) I0324 15:42:54.806144 1 gather_workloads_info.go:387] No image sha256:07f14370b57046dc2ac49be933bb2248aaf5943e8fb56ce9703eaa7c673b999f (100ms) I0324 15:42:54.906438 1 gather_workloads_info.go:398] Found image sha256:670004bffddb41c787b9f9a5c7d1615453d4acfa6452dedfa61ad2f305eea814 (100ms) I0324 15:42:55.005652 1 gather_workloads_info.go:387] No image sha256:2962c6c31d7078b34cf7ba0c9b0ea685dc539cea67b06bd38b6ac566298bb790 (99ms) I0324 15:42:55.107795 1 gather_workloads_info.go:387] No image sha256:7ea1ff613255c1ee67b34ec82b68ec6f5a4a6d283620a8e0ad142d4a31eb9211 (102ms) I0324 15:42:55.207352 1 gather_workloads_info.go:387] No image sha256:11a864539ae77a94afcf8d2e1abf9168081c5081a1bc06088f4be5206cdc02df (100ms) I0324 15:42:55.306966 1 gather_workloads_info.go:387] No image sha256:2a4f6a54cd5135c8121832236063419071fa5d00c10a051c197d3d01370e4d9c (100ms) I0324 15:42:55.405680 1 gather_workloads_info.go:387] No image sha256:04ee35aa6f87e954d386228808017cf2938c5a96a052059c6767078d5f3ad8d6 (99ms) I0324 15:42:55.505595 1 gather_workloads_info.go:387] No image sha256:a9dc28a6962d195dcce0c76a544aad5f784dd90b98282c169d7b94c870a85f8b (100ms) I0324 15:42:55.606511 1 gather_workloads_info.go:387] No image sha256:41e0dcd2f91df9234249f168b3fbe6d55cfdb3ebd2e598d5edfa47f905c046cb (101ms) I0324 15:42:55.706148 1 gather_workloads_info.go:387] No image sha256:317131706dcce9c87eb4e1efb8446bdbf5e0726e120a8f6c655e2acd89d60e2d (100ms) I0324 15:42:55.806197 1 gather_workloads_info.go:387] No image sha256:7db22f56cf9d223b1437e8cba3aa9e04e8ae8b973e23e85133bf1d3d8d66f15e (100ms) I0324 15:42:55.906105 1 gather_workloads_info.go:387] No image sha256:2af652f68b17ed49c2565928e4c101880e949c4484c6ee958793a8e668e47bea (100ms) I0324 15:42:56.007396 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (101ms) I0324 15:42:56.107976 1 gather_workloads_info.go:387] No image sha256:04c7a2673571e9069c934d86b4308c0f57b2715b45db500837b7708ae1f20ae7 (101ms) I0324 15:42:56.205767 1 gather_workloads_info.go:387] No image sha256:1bf773bc4ec02bf6d68c82f32175366b01fb24f6da8c724b7621748692aec7dc (98ms) I0324 15:42:56.308855 1 gather_workloads_info.go:387] No image sha256:1ff360b712000d39cbb58158186ff0a18764278e6fa98ef9f21d113b978d9a79 (103ms) I0324 15:42:56.406116 1 gather_workloads_info.go:387] No image sha256:3e262d5207d5e2077e4a14f4fbddc70d4d2651dbaaf955147013feb58a18cddd (97ms) I0324 15:42:56.505940 1 gather_workloads_info.go:387] No image sha256:8f13a1e673c882ee926e97270249a20d3c40797a8c7c5abb2fda9c98c24a2eb4 (100ms) I0324 15:42:56.605139 1 gather_workloads_info.go:387] No image sha256:05376e7e2304d22dd866d8e25dd1e5ff8988b2232059fadfd12cb440af785635 (99ms) I0324 15:42:56.705957 1 gather_workloads_info.go:387] No image sha256:124f1b391b0f376a5deffd05b29ef0896ca0912e1f67b3a2051ed6678c54a57e (101ms) I0324 15:42:56.705990 1 tasks_processing.go:74] worker 1 stopped. I0324 15:42:56.706356 1 recorder.go:75] Recording config/workload_info with fingerprint=1e0927cac0352232039218041545be3fc8fa37e099e6e02318f85a2f74513863 I0324 15:42:56.706372 1 gather.go:177] gatherer "workloads" function "workload_info" took 9.52491687s to process 1 records I0324 15:42:56.706386 1 periodic.go:263] Periodic gather workloads completed in 9.525s I0324 15:42:56.706393 1 controllerstatus.go:80] name=periodic-workloads healthy=true reason= message= I0324 15:42:56.706398 1 periodic.go:216] Running conditional gatherer I0324 15:42:56.712438 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.21.5/gathering_rules I0324 15:42:56.881412 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer E0324 15:42:56.883791 1 conditional_gatherer.go:324] unable to update alerts cache: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?match%5B%5D=ALERTS%7Balertstate%3D%22firing%22%7D&query=ALERTS": dial tcp 172.30.208.16:9091: connect: connection refused I0324 15:42:56.883865 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0324 15:42:56.889398 1 conditional_gatherer.go:392] cluster version is '4.21.5' E0324 15:42:56.889413 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0324 15:42:56.889419 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0324 15:42:56.889422 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0324 15:42:56.889425 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0324 15:42:56.889428 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0324 15:42:56.889430 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0324 15:42:56.889433 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0324 15:42:56.889436 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0324 15:42:56.889438 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0324 15:42:56.889441 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0324 15:42:56.889455 1 tasks_processing.go:45] number of workers: 3 I0324 15:42:56.889466 1 tasks_processing.go:69] worker 2 listening for tasks. I0324 15:42:56.889469 1 tasks_processing.go:71] worker 2 working on remote_configuration task. I0324 15:42:56.889476 1 tasks_processing.go:69] worker 0 listening for tasks. I0324 15:42:56.889492 1 tasks_processing.go:69] worker 1 listening for tasks. I0324 15:42:56.889499 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0324 15:42:56.889503 1 tasks_processing.go:74] worker 1 stopped. I0324 15:42:56.889494 1 tasks_processing.go:71] worker 0 working on rapid_container_logs task. I0324 15:42:56.889516 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0324 15:42:56.889527 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.141µs to process 1 records I0324 15:42:56.889584 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=9fafdcbf373f1f367f1d69551ea59114592cce1681189be0f3dde774e0530661 I0324 15:42:56.889593 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.76µs to process 1 records I0324 15:42:56.889599 1 tasks_processing.go:74] worker 2 stopped. I0324 15:42:56.889755 1 tasks_processing.go:74] worker 0 stopped. I0324 15:42:56.889767 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 237.115µs to process 0 records I0324 15:42:56.889782 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0324 15:42:56.889799 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0324 15:42:56.917513 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=28a1bb68410f504bf9716f7e83636cf65533705318255f76c500b72d29c7fc45 I0324 15:42:56.917634 1 diskrecorder.go:70] Writing 110 records to /var/lib/insights-operator/insights-2026-03-24-154256.tar.gz I0324 15:42:56.924863 1 diskrecorder.go:51] Wrote 110 records to disk in 7ms I0324 15:42:56.924893 1 periodic.go:285] Gathering cluster info every 2h0m0s I0324 15:42:56.924907 1 periodic.go:286] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0324 15:44:03.562970 1 diskrecorder.go:223] Found files to send: insights-2026-03-24-154256.tar.gz I0324 15:44:03.563000 1 insightsuploader.go:150] Checking archives to upload periodically every 15m40.233088086s I0324 15:44:03.563008 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0324 15:44:03.574334 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0324 15:44:03.746697 1 requests.go:87] Successfully reported id=2026-03-24T15:44:03Z x-rh-insights-request-id=5bab22d340d74a65a84f0e63836eda53, wrote=51693 I0324 15:44:03.746754 1 insightsuploader.go:187] Uploaded report successfully in 183.737067ms I0324 15:44:03.746779 1 controller.go:129] Initializing last reported time to 2026-03-24T15:44:03Z I0324 15:44:03.746837 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0324 15:44:03.746846 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0324 15:44:03.746852 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0324 15:44:03.752972 1 controller.go:554] The operator is healthy I0324 15:44:33.594953 1 controller.go:554] The operator is healthy I0324 15:45:04.216651 1 insightsreport.go:137] Pulling report from smart-proxy I0324 15:45:04.216678 1 insightsreport.go:149] Retrieving report I0324 15:45:04.224153 1 requests.go:110] Retrieving report for cluster: e8511527-61b3-4f6b-99d0-3c856d26357b I0324 15:45:04.224167 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/e8511527-61b3-4f6b-99d0-3c856d26357b/reports I0324 15:45:04.230914 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/e8511527-61b3-4f6b-99d0-3c856d26357b/reports I0324 15:45:04.663121 1 insightsreport.go:184] Report retrieved I0324 15:45:04.688743 1 insightsreport.go:239] Report retrieved correctly I0324 15:46:33.597895 1 controller.go:554] The operator is healthy I0324 15:47:33.550297 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0324 15:47:33.556812 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0324 15:47:33.556835 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0324 15:47:33.561081 1 secretconfigobserver.go:119] support secret does not exist I0324 15:48:33.592092 1 controller.go:554] The operator is healthy I0324 15:50:33.591151 1 controller.go:554] The operator is healthy I0324 15:52:33.547303 1 configmapobserver.go:84] configmaps "insights-config" not found I0324 15:52:33.551920 1 configmapobserver.go:84] configmaps "insights-config" not found I0324 15:52:33.562168 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0324 15:52:33.567375 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0324 15:52:33.567391 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0324 15:52:33.571624 1 secretconfigobserver.go:119] support secret does not exist I0324 15:52:33.591658 1 controller.go:554] The operator is healthy I0324 15:54:33.592468 1 controller.go:554] The operator is healthy I0324 15:56:33.590884 1 controller.go:554] The operator is healthy I0324 15:57:33.572152 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0324 15:57:33.579231 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0324 15:57:33.579253 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0324 15:57:33.583520 1 secretconfigobserver.go:119] support secret does not exist I0324 15:58:33.592214 1 controller.go:554] The operator is healthy I0324 15:59:48.981471 1 insightsuploader.go:146] Nothing to report since 2026-03-24T15:44:03Z I0324 16:00:33.594463 1 controller.go:554] The operator is healthy I0324 16:02:33.547246 1 configmapobserver.go:84] configmaps "insights-config" not found I0324 16:02:33.584454 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0324 16:02:33.589167 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0324 16:02:33.589190 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0324 16:02:33.590977 1 controller.go:554] The operator is healthy I0324 16:02:33.593334 1 secretconfigobserver.go:119] support secret does not exist I0324 16:04:33.591777 1 controller.go:554] The operator is healthy I0324 16:06:33.591905 1 controller.go:554] The operator is healthy I0324 16:07:33.593376 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0324 16:07:33.599357 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0324 16:07:33.599378 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0324 16:07:33.604003 1 secretconfigobserver.go:119] support secret does not exist