I0705 21:52:28.568513 1 cmd.go:253] Using service-serving-cert provided certificates I0705 21:52:28.570227 1 observer_polling.go:159] Starting file observer I0705 21:52:29.093305 1 operator.go:60] Starting insights-operator v0.0.0-master+$Format:%H$ I0705 21:52:29.093532 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0705 21:52:29.094055 1 secure_serving.go:57] Forcing use of http/1.1 only W0705 21:52:29.094078 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0705 21:52:29.094086 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0705 21:52:29.094095 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0705 21:52:29.094100 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0705 21:52:29.094104 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0705 21:52:29.094107 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0705 21:52:29.094220 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0705 21:52:29.099095 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"6017f7a4-3b5c-4b6f-9052-acd0955cba3a", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}} I0705 21:52:29.099087 1 operator.go:125] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GatewayAPI GatewayAPIController HighlyAvailableArbiter ImageVolume IngressControllerLBSubnetsAWS KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController SigstoreImageVerification StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPClusterHostedDNSInstall GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall ImageModeStatusReporting ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesAzure ManagedBootImagesvSphere MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutatingAdmissionPolicy NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterOperations NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PreconfiguredUDNAddresses SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeAttributesClass VolumeGroupSnapshot] I0705 21:52:29.100398 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0705 21:52:29.100412 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0705 21:52:29.100415 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0705 21:52:29.100416 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0705 21:52:29.100432 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0705 21:52:29.100432 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0705 21:52:29.100738 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0705 21:52:29.101132 1 secure_serving.go:213] Serving securely on [::]:8443 I0705 21:52:29.101163 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0705 21:52:29.105584 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0705 21:52:29.105609 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0705 21:52:29.105702 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0705 21:52:29.112978 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0705 21:52:29.113043 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0705 21:52:29.120876 1 secretconfigobserver.go:119] support secret does not exist I0705 21:52:29.127161 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0705 21:52:29.135032 1 secretconfigobserver.go:119] support secret does not exist I0705 21:52:29.136279 1 recorder.go:161] Pruning old reports every 8h19m51s, max age is 288h0m0s I0705 21:52:29.142665 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0705 21:52:29.142683 1 periodic.go:209] Running clusterconfig gatherer I0705 21:52:29.142684 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0705 21:52:29.142723 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0705 21:52:29.142727 1 tasks_processing.go:45] number of workers: 32 I0705 21:52:29.142732 1 insightsreport.go:296] Starting report retriever I0705 21:52:29.142738 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0705 21:52:29.142752 1 tasks_processing.go:69] worker 0 listening for tasks. I0705 21:52:29.142754 1 tasks_processing.go:69] worker 2 listening for tasks. I0705 21:52:29.142762 1 tasks_processing.go:71] worker 0 working on storage_classes task. I0705 21:52:29.142772 1 tasks_processing.go:69] worker 10 listening for tasks. I0705 21:52:29.142767 1 tasks_processing.go:69] worker 1 listening for tasks. I0705 21:52:29.142780 1 tasks_processing.go:71] worker 10 working on jaegers task. I0705 21:52:29.142773 1 tasks_processing.go:69] worker 24 listening for tasks. I0705 21:52:29.142763 1 tasks_processing.go:69] worker 16 listening for tasks. I0705 21:52:29.142782 1 tasks_processing.go:69] worker 17 listening for tasks. I0705 21:52:29.142789 1 tasks_processing.go:69] worker 18 listening for tasks. I0705 21:52:29.142796 1 tasks_processing.go:69] worker 19 listening for tasks. I0705 21:52:29.142797 1 tasks_processing.go:69] worker 13 listening for tasks. I0705 21:52:29.142803 1 tasks_processing.go:69] worker 5 listening for tasks. I0705 21:52:29.142807 1 tasks_processing.go:69] worker 20 listening for tasks. I0705 21:52:29.142787 1 tasks_processing.go:69] worker 9 listening for tasks. I0705 21:52:29.142812 1 tasks_processing.go:69] worker 11 listening for tasks. I0705 21:52:29.142814 1 tasks_processing.go:69] worker 3 listening for tasks. I0705 21:52:29.142810 1 tasks_processing.go:69] worker 6 listening for tasks. I0705 21:52:29.142820 1 tasks_processing.go:69] worker 22 listening for tasks. I0705 21:52:29.142822 1 tasks_processing.go:69] worker 12 listening for tasks. I0705 21:52:29.142816 1 tasks_processing.go:69] worker 21 listening for tasks. I0705 21:52:29.142827 1 tasks_processing.go:69] worker 4 listening for tasks. I0705 21:52:29.142829 1 tasks_processing.go:69] worker 8 listening for tasks. I0705 21:52:29.142820 1 tasks_processing.go:69] worker 7 listening for tasks. I0705 21:52:29.142837 1 tasks_processing.go:69] worker 26 listening for tasks. I0705 21:52:29.142835 1 tasks_processing.go:69] worker 28 listening for tasks. I0705 21:52:29.142842 1 tasks_processing.go:69] worker 15 listening for tasks. I0705 21:52:29.142842 1 tasks_processing.go:69] worker 27 listening for tasks. I0705 21:52:29.142845 1 tasks_processing.go:69] worker 25 listening for tasks. I0705 21:52:29.142849 1 tasks_processing.go:69] worker 31 listening for tasks. I0705 21:52:29.142849 1 tasks_processing.go:69] worker 30 listening for tasks. I0705 21:52:29.142835 1 tasks_processing.go:69] worker 14 listening for tasks. I0705 21:52:29.142853 1 tasks_processing.go:69] worker 29 listening for tasks. I0705 21:52:29.142827 1 tasks_processing.go:69] worker 23 listening for tasks. I0705 21:52:29.142860 1 tasks_processing.go:71] worker 16 working on infrastructures task. I0705 21:52:29.142862 1 tasks_processing.go:71] worker 1 working on container_runtime_configs task. I0705 21:52:29.142866 1 tasks_processing.go:71] worker 23 working on container_images task. I0705 21:52:29.142899 1 tasks_processing.go:71] worker 24 working on machine_autoscalers task. I0705 21:52:29.142763 1 tasks_processing.go:71] worker 2 working on mutating_webhook_configurations task. I0705 21:52:29.143037 1 tasks_processing.go:71] worker 21 working on install_plans task. I0705 21:52:29.143049 1 tasks_processing.go:71] worker 7 working on machine_config_pools task. I0705 21:52:29.143099 1 tasks_processing.go:71] worker 20 working on nodenetworkconfigurationpolicies task. I0705 21:52:29.143149 1 tasks_processing.go:71] worker 18 working on support_secret task. I0705 21:52:29.143240 1 tasks_processing.go:71] worker 28 working on operators task. I0705 21:52:29.143287 1 tasks_processing.go:71] worker 22 working on node_logs task. I0705 21:52:29.143327 1 tasks_processing.go:71] worker 30 working on networks task. I0705 21:52:29.143382 1 tasks_processing.go:71] worker 29 working on ingress_certificates task. I0705 21:52:29.143103 1 tasks_processing.go:71] worker 3 working on pdbs task. I0705 21:52:29.143043 1 tasks_processing.go:71] worker 4 working on proxies task. I0705 21:52:29.143039 1 tasks_processing.go:71] worker 15 working on nodenetworkstates task. I0705 21:52:29.143108 1 tasks_processing.go:71] worker 9 working on certificate_signing_requests task. I0705 21:52:29.143113 1 tasks_processing.go:71] worker 11 working on authentication task. I0705 21:52:29.143284 1 tasks_processing.go:71] worker 13 working on openshift_logging task. I0705 21:52:29.143122 1 tasks_processing.go:71] worker 31 working on openstack_dataplanedeployments task. I0705 21:52:29.143126 1 tasks_processing.go:71] worker 27 working on clusterroles task. I0705 21:52:29.143130 1 tasks_processing.go:71] worker 25 working on feature_gates task. I0705 21:52:29.143136 1 tasks_processing.go:71] worker 26 working on monitoring_persistent_volumes task. I0705 21:52:29.143142 1 tasks_processing.go:71] worker 19 working on machine_configs task. I0705 21:52:29.143146 1 tasks_processing.go:71] worker 17 working on sap_datahubs task. I0705 21:52:29.143301 1 tasks_processing.go:71] worker 5 working on openshift_machine_api_events task. I0705 21:52:29.143308 1 tasks_processing.go:71] worker 6 working on active_alerts task. I0705 21:52:29.143317 1 tasks_processing.go:71] worker 12 working on sap_config task. I0705 21:52:29.143323 1 tasks_processing.go:71] worker 14 working on qemu_kubevirt_launcher_logs task. I0705 21:52:29.143046 1 tasks_processing.go:71] worker 8 working on openstack_controlplanes task. I0705 21:52:29.148929 1 tasks_processing.go:71] worker 10 working on openstack_version task. I0705 21:52:29.148946 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 6.134343ms to process 0 records E0705 21:52:29.149152 1 gather_active_alerts.go:64] Unable to retrieve most recent alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp 172.30.220.89:9094: connect: connection refused I0705 21:52:29.149196 1 tasks_processing.go:71] worker 6 working on storage_cluster task. E0705 21:52:29.149207 1 gather.go:140] gatherer "clusterconfig" function "active_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp 172.30.220.89:9094: connect: connection refused I0705 21:52:29.149230 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 4.78279ms to process 0 records I0705 21:52:29.149389 1 tasks_processing.go:71] worker 1 working on machine_healthchecks task. I0705 21:52:29.149414 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 6.518354ms to process 0 records I0705 21:52:29.149661 1 tasks_processing.go:71] worker 24 working on nodes task. I0705 21:52:29.149674 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 6.724175ms to process 0 records I0705 21:52:29.150399 1 tasks_processing.go:71] worker 13 working on image_pruners task. I0705 21:52:29.150412 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 6.610995ms to process 0 records I0705 21:52:29.150475 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 7.319789ms to process 0 records I0705 21:52:29.150483 1 tasks_processing.go:71] worker 20 working on olm_operators task. I0705 21:52:29.150704 1 tasks_processing.go:71] worker 31 working on ingress task. I0705 21:52:29.150713 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 6.809332ms to process 0 records I0705 21:52:29.151436 1 tasks_processing.go:71] worker 17 working on tsdb_status task. I0705 21:52:29.151454 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 7.238736ms to process 0 records I0705 21:52:29.151515 1 tasks_processing.go:71] worker 12 working on pod_network_connectivity_checks task. I0705 21:52:29.151533 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 7.009546ms to process 0 records I0705 21:52:29.151954 1 tasks_processing.go:71] worker 15 working on cluster_apiserver task. I0705 21:52:29.151967 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 8.377007ms to process 0 records I0705 21:52:29.152567 1 tasks_processing.go:71] worker 8 working on image_registries task. I0705 21:52:29.152586 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 7.884392ms to process 0 records I0705 21:52:29.152945 1 tasks_processing.go:71] worker 18 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. E0705 21:52:29.152990 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0705 21:52:29.153042 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 9.776444ms to process 0 records I0705 21:52:29.153190 1 tasks_processing.go:71] worker 4 working on oauths task. I0705 21:52:29.153360 1 recorder.go:75] Recording config/proxy with fingerprint=0fcc52b5f1c4b78768e3c4f0a140a319a5784a8c92ea522950b9e593b8082678 I0705 21:52:29.153382 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 9.625875ms to process 1 records I0705 21:52:29.154919 1 tasks_processing.go:71] worker 10 working on validating_webhook_configurations task. I0705 21:52:29.154929 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 5.942751ms to process 0 records I0705 21:52:29.155192 1 tasks_processing.go:71] worker 6 working on crds task. I0705 21:52:29.155361 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 5.899328ms to process 0 records I0705 21:52:29.155650 1 tasks_processing.go:71] worker 1 working on overlapping_namespace_uids task. E0705 21:52:29.155666 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0705 21:52:29.155682 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 6.240109ms to process 0 records I0705 21:52:29.155814 1 controller.go:128] Initializing last reported time to 0001-01-01T00:00:00Z I0705 21:52:29.155858 1 controller.go:326] The initial operator extension status is healthy I0705 21:52:29.155874 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0705 21:52:29.155884 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0705 21:52:29.155890 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0705 21:52:29.155922 1 controller.go:489] The operator is still being initialized I0705 21:52:29.155934 1 controller.go:512] The operator is healthy I0705 21:52:29.156018 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0705 21:52:29.156070 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0705 21:52:29.156402 1 operator.go:288] started I0705 21:52:29.156528 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer E0705 21:52:29.158701 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0705 21:52:29.158722 1 tasks_processing.go:71] worker 17 working on service_accounts task. E0705 21:52:29.158798 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0705 21:52:29.159249 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 7.232958ms to process 0 records I0705 21:52:29.159271 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 15.902262ms to process 0 records I0705 21:52:29.159293 1 tasks_processing.go:71] worker 7 working on operators_pods_and_events task. I0705 21:52:29.159354 1 tasks_processing.go:71] worker 16 working on config_maps task. I0705 21:52:29.159928 1 recorder.go:75] Recording config/infrastructure with fingerprint=ad46c5bbc7d9603e33c2e03773ed606fc64088863cb60deb4c0e5c05ab013c65 I0705 21:52:29.159946 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 16.274986ms to process 1 records I0705 21:52:29.159963 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 14.961624ms to process 0 records I0705 21:52:29.159983 1 tasks_processing.go:71] worker 5 working on schedulers task. I0705 21:52:29.160038 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=68482c42aa9743bf38e86060b145593246a5f39a2db17348d85cf526b07b75d5 I0705 21:52:29.160084 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=6db71267de4119d99a8b21267b31e2c97295614d1ea5ca2133ab75bdd8ad3ff9 I0705 21:52:29.160095 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 16.54313ms to process 2 records I0705 21:52:29.160064 1 tasks_processing.go:71] worker 0 working on silenced_alerts task. I0705 21:52:29.160199 1 tasks_processing.go:71] worker 11 working on machine_sets task. I0705 21:52:29.160367 1 recorder.go:75] Recording config/authentication with fingerprint=b726085f73f2e5c9b381384c68dd4798b5b87f0767af03615793a38fbea6d4fb I0705 21:52:29.160380 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 15.595027ms to process 1 records I0705 21:52:29.160487 1 tasks_processing.go:71] worker 30 working on cost_management_metrics_configs task. I0705 21:52:29.160530 1 recorder.go:75] Recording config/network with fingerprint=d9a4b6229e8a507a57d19a237b3a3fc26f6c8f830722d359d77c79621b368d73 I0705 21:52:29.160540 1 gather.go:177] gatherer "clusterconfig" function "networks" took 15.932214ms to process 1 records I0705 21:52:29.160620 1 tasks_processing.go:71] worker 25 working on image task. I0705 21:52:29.160643 1 recorder.go:75] Recording config/featuregate with fingerprint=4c7bc493bd30e4b27fe48dcdedf88bb7019677fff2c36138e4b6e804fa6be32e I0705 21:52:29.160749 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 16.498103ms to process 1 records I0705 21:52:29.160981 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 17.630787ms to process 0 records I0705 21:52:29.161027 1 tasks_processing.go:71] worker 22 working on aggregated_monitoring_cr_names task. I0705 21:52:29.161032 1 tasks_processing.go:71] worker 3 working on version task. I0705 21:52:29.161233 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=a9c1aa3510c59cb110dfba13159fc3840aee44b17e15e568123951775be31f70 I0705 21:52:29.161281 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=b15ca43991632ee1b128f6f65ffae8a5d9db3498bc256ce6ea44a1d927d82f09 I0705 21:52:29.161307 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=0e59c40e4687b6b19b081e9064e95c434da52ec8e425e041b81a7d266b1d8853 I0705 21:52:29.161328 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=acf9b1284b9564db1599e4fefced2daffdf81c73fe84a6b8129981005fd31186 I0705 21:52:29.161377 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=28c65218ffe43f714cef821757d6779d9b9f201d2cfc02344f47aad1910cfa16 I0705 21:52:29.161433 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=293af664d0a32f63cfac9904e651c8498a51199e9741d78b93a591efae89c4bd I0705 21:52:29.161471 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=9bb206acc2e15d66a2654f78666ae6b7b612b8139132769134e03d6b60ee9ae2 I0705 21:52:29.161484 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 17.534725ms to process 7 records E0705 21:52:29.162551 1 gather_silenced_alerts.go:52] Unable to retrieve silenced alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp 172.30.220.89:9094: connect: connection refused I0705 21:52:29.162573 1 tasks_processing.go:71] worker 0 working on metrics task. E0705 21:52:29.162590 1 gather.go:140] gatherer "clusterconfig" function "silenced_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp 172.30.220.89:9094: connect: connection refused I0705 21:52:29.162613 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 2.426158ms to process 0 records E0705 21:52:29.164959 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0705 21:52:29.164983 1 tasks_processing.go:71] worker 0 working on ceph_cluster task. E0705 21:52:29.164998 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0705 21:52:29.165024 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 2.396185ms to process 0 records I0705 21:52:29.180707 1 gather_logs.go:145] no pods in namespace were found I0705 21:52:29.180769 1 tasks_processing.go:71] worker 14 working on machines task. I0705 21:52:29.180780 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 36.164341ms to process 0 records I0705 21:52:29.180942 1 tasks_processing.go:71] worker 2 working on openstack_dataplanenodesets task. I0705 21:52:29.181333 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=5635ef57ae3eba6387c4525e8b13eee829432fdee493779421223513125aa0ce I0705 21:52:29.181443 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=2c513f29409f3441f9ce04ffe3199478318ea491ca5c6e6ed5db3fece87b6077 I0705 21:52:29.181482 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=bcbdc9379c6e7c1db8442e2833f31a713f1a1b21c18ff5262f01d7803f1f68ba I0705 21:52:29.181491 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 37.909058ms to process 3 records I0705 21:52:29.181592 1 tasks_processing.go:71] worker 4 working on dvo_metrics task. I0705 21:52:29.181724 1 recorder.go:75] Recording config/oauth with fingerprint=a178b18203401c1ab88c9d10da7b2898059d884548bfeba8f45f5246a773c6d4 I0705 21:52:29.181734 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 27.846961ms to process 1 records I0705 21:52:29.181782 1 tasks_processing.go:71] worker 13 working on lokistack task. I0705 21:52:29.181992 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=ac3ff9c0a940fca23828e72e3e33a25fa78e2ad8b55bd7cf393c5a32a9037c3d I0705 21:52:29.182094 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 31.07815ms to process 1 records I0705 21:52:29.182108 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 37.48648ms to process 0 records I0705 21:52:29.182114 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 31.576451ms to process 0 records I0705 21:52:29.182122 1 tasks_processing.go:71] worker 20 working on sap_pods task. I0705 21:52:29.182152 1 tasks_processing.go:74] worker 26 stopped. I0705 21:52:29.182236 1 tasks_processing.go:74] worker 12 stopped. E0705 21:52:29.182251 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0705 21:52:29.182262 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 30.705145ms to process 0 records I0705 21:52:29.182359 1 tasks_processing.go:74] worker 15 stopped. I0705 21:52:29.182440 1 recorder.go:75] Recording config/apiserver with fingerprint=28c8572d7f25d651a4b45fbeac5887c48599c9d229ac08b3f7e8f381a8b27cb5 I0705 21:52:29.182457 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 30.263949ms to process 1 records I0705 21:52:29.186207 1 tasks_processing.go:74] worker 11 stopped. I0705 21:52:29.186247 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 25.990671ms to process 0 records I0705 21:52:29.186718 1 tasks_processing.go:74] worker 5 stopped. I0705 21:52:29.186864 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=38c3d81a1bea2ca9dab456969ec9ddb6d2f3c1410c2946879e1c27229ddcae06 I0705 21:52:29.186901 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 26.718793ms to process 1 records I0705 21:52:29.188286 1 tasks_processing.go:74] worker 25 stopped. I0705 21:52:29.189055 1 recorder.go:75] Recording config/image with fingerprint=e74c29d636610a12b30bc0d2404431dbd4f8ce6896cf819ba7301046a3bb8b31 I0705 21:52:29.189119 1 gather.go:177] gatherer "clusterconfig" function "image" took 27.624398ms to process 1 records I0705 21:52:29.189277 1 tasks_processing.go:74] worker 31 stopped. I0705 21:52:29.189390 1 recorder.go:75] Recording config/ingress with fingerprint=a1002af2851189a0475de8162c606e2c13e939ed9cb1b9e77d832a5c2008595b I0705 21:52:29.189438 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 38.300686ms to process 1 records I0705 21:52:29.189552 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=1ece66aa6d3e1e2da7ac5cc0bc4cf161dac403604b03fe2a8a73366700e8acc6 I0705 21:52:29.189647 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=8bf6cbd36d5a2278f001a7719b7fc8fcf56f6c33c58c51d901399cbdd7914fd3 I0705 21:52:29.189800 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=5b15c5f07e053246c0453547d90ca58408b3c1be577f0c3cb593e3b46916eb10 I0705 21:52:29.189874 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=8a54e5033fc06af13eadaa920fb845487a37fa52838589fa39e559e1357f647b I0705 21:52:29.189936 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=5ae2b7fa3fd2fe090f3e35d7d952dd1fbf19909dcd194a616ea2893f57b13fb4 I0705 21:52:29.190006 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=3623e6d18a8fbaf56e9bbad89455c28d40c731199dae42fa567555208d6770bc I0705 21:52:29.190088 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=cf99dc0b4de1264773c6083960f056ed7da06f64da3cc7210ae37e3ba098d55b I0705 21:52:29.190153 1 tasks_processing.go:74] worker 10 stopped. I0705 21:52:29.190179 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=e4e9eb330e95244a382795c96608232d4d26b00503bc9f8e66c3b1da36d26c4d I0705 21:52:29.190324 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=b51054f8ee5e4a0bdc1129e1d1af42dd80c8155e80b8f238af4e8f211d3fdb6f I0705 21:52:29.190087 1 controller.go:212] Source scaController *sca.Controller is not ready I0705 21:52:29.190381 1 controller.go:212] Source clusterTransferController *clustertransfer.Controller is not ready I0705 21:52:29.190388 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0705 21:52:29.190394 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0705 21:52:29.190397 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0705 21:52:29.190422 1 controller.go:489] The operator is still being initialized I0705 21:52:29.190433 1 controller.go:512] The operator is healthy I0705 21:52:29.190486 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=a401ef87f336466151f19fcb397e407af3b298ffe0e915ca7498aa1f2af4e156 I0705 21:52:29.190583 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=d5de2627a6e66c34d7085f6747a26d04deed8d28689afba8a029c09ff4af64de I0705 21:52:29.190663 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=a649061af97613706214c660acd5763e7e18d6fc80a353d9593b5eb5f73a65f0 I0705 21:52:29.190755 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=f58e868496db3ddabc490e020e63e705b1878bedbb1757093440fa8a5ce260a4 I0705 21:52:29.190789 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 34.102286ms to process 13 records I0705 21:52:29.190821 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 45.459345ms to process 0 records I0705 21:52:29.190847 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 28.727594ms to process 0 records I0705 21:52:29.190877 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 24.268804ms to process 0 records I0705 21:52:29.191334 1 recorder.go:75] Recording config/node/ip-10-0-0-35.ec2.internal with fingerprint=df899b60d6549a79e843f02a2571c842052b4797e246754764c4dce22f9fff54 I0705 21:52:29.191459 1 recorder.go:75] Recording config/node/ip-10-0-2-82.ec2.internal with fingerprint=79ca1f338b9f8397a42a180aaf592ffd8a1ceb5a61110289aa393aa215f35a63 I0705 21:52:29.191489 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 39.993462ms to process 2 records I0705 21:52:29.191571 1 tasks_processing.go:74] worker 30 stopped. I0705 21:52:29.191594 1 tasks_processing.go:74] worker 9 stopped. I0705 21:52:29.191600 1 tasks_processing.go:74] worker 0 stopped. I0705 21:52:29.191624 1 tasks_processing.go:74] worker 24 stopped. I0705 21:52:29.191809 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=7e35d125942720459ea0f1501425cb679984aca23fcc6f583022ef85f1a8e14d I0705 21:52:29.191875 1 tasks_processing.go:74] worker 27 stopped. I0705 21:52:29.191957 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=c2fe5e4b77c1d5c5df893b32ed3d5517842f9e4adfade7f2a519c01715f6c215 I0705 21:52:29.191989 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 46.392841ms to process 2 records E0705 21:52:29.192017 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0705 21:52:29.192049 1 gather.go:177] gatherer "clusterconfig" function "machines" took 10.051125ms to process 0 records I0705 21:52:29.192070 1 tasks_processing.go:74] worker 14 stopped. I0705 21:52:29.192076 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 9.870815ms to process 0 records I0705 21:52:29.192096 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 9.099326ms to process 0 records I0705 21:52:29.192102 1 tasks_processing.go:74] worker 2 stopped. I0705 21:52:29.192145 1 tasks_processing.go:74] worker 13 stopped. I0705 21:52:29.192193 1 tasks_processing.go:74] worker 8 stopped. I0705 21:52:29.192610 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=45ec586978ef0754deca14b27ff3ce434394533858a53965ddd8a63808432a0a I0705 21:52:29.192627 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 38.352817ms to process 1 records I0705 21:52:29.192643 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0705 21:52:29.192650 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 35.903702ms to process 1 records I0705 21:52:29.192656 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 9.908427ms to process 0 records I0705 21:52:29.192665 1 tasks_processing.go:74] worker 20 stopped. I0705 21:52:29.192670 1 tasks_processing.go:74] worker 1 stopped. I0705 21:52:29.195603 1 tasks_processing.go:74] worker 7 stopped. I0705 21:52:29.195630 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 36.296129ms to process 0 records I0705 21:52:29.198468 1 tasks_processing.go:74] worker 19 stopped. I0705 21:52:29.198561 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0705 21:52:29.198581 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 54.30644ms to process 1 records W0705 21:52:29.199993 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0705 21:52:29.200455 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0705 21:52:29.200468 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0705 21:52:29.200539 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0705 21:52:29.206461 1 base_controller.go:82] Caches are synced for ConfigController I0705 21:52:29.206477 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0705 21:52:29.210013 1 tasks_processing.go:74] worker 6 stopped. I0705 21:52:29.210784 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=084446d9ac8bc40787ea308bfbf1225255d82bb324bef50386aec08273fc9488 I0705 21:52:29.211108 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=8a59835e15b1a76b002bd295b8de7eed1f8448375825c4743bc50c85a1995c68 I0705 21:52:29.211124 1 gather.go:177] gatherer "clusterconfig" function "crds" took 54.800021ms to process 2 records I0705 21:52:29.219013 1 configmapobserver.go:84] configmaps "insights-config" not found I0705 21:52:29.220004 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0705 21:52:29.220720 1 tasks_processing.go:74] worker 22 stopped. I0705 21:52:29.220735 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 59.675472ms to process 0 records I0705 21:52:29.240138 1 tasks_processing.go:74] worker 23 stopped. I0705 21:52:29.243016 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-575cd97545-b5tbd with fingerprint=d0e496699165cbbf36c647f58470290be88ae4a01e570d2505b559f5bec5e4ec I0705 21:52:29.243142 1 recorder.go:75] Recording config/running_containers with fingerprint=c0cc9da0a30072aa6dfb72027b8ab724005ed94aea5ad7ab18fc7b211e6a42e5 I0705 21:52:29.243157 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 97.253741ms to process 2 records I0705 21:52:29.246572 1 tasks_processing.go:74] worker 16 stopped. E0705 21:52:29.246588 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0705 21:52:29.246597 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0705 21:52:29.246602 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0705 21:52:29.246614 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=26b6661162b099a0f5a279859b4f46c867929a79d9a4a41fde4be4e6fe138018 I0705 21:52:29.246646 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=f73fcd48d6f9ebbc83253a1b5f23ef59d0683465ec46e474389103c717467dfb I0705 21:52:29.246653 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0705 21:52:29.246657 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=c93090eb0d2a4736885abeb79c91680cfd01fda46464f83456b085d4dc8239f0 I0705 21:52:29.246666 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0705 21:52:29.246703 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0705 21:52:29.246711 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0705 21:52:29.246716 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 87.194028ms to process 7 records I0705 21:52:29.246727 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 93.724287ms to process 0 records I0705 21:52:29.246733 1 tasks_processing.go:74] worker 18 stopped. I0705 21:52:29.256621 1 base_controller.go:82] Caches are synced for LoggingSyncer I0705 21:52:29.256635 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0705 21:52:29.260495 1 tasks_processing.go:74] worker 3 stopped. I0705 21:52:29.260781 1 recorder.go:75] Recording config/version with fingerprint=f9e1530a1a6b73534c643dd49be74697654f14a8e3b73edd00ffb01a2a77f344 I0705 21:52:29.260793 1 recorder.go:75] Recording config/id with fingerprint=fb9082caf345808f8b8406ccb1f0dc3d02b3b88b5fdb631dce0b9de8c257cff4 I0705 21:52:29.260800 1 gather.go:177] gatherer "clusterconfig" function "version" took 99.450523ms to process 2 records I0705 21:52:29.267540 1 cluster_transfer.go:102] no available accepted cluster transfer I0705 21:52:29.267557 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0705 21:52:29.284687 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0705 21:52:29.290821 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0705 21:52:29.297694 1 tasks_processing.go:74] worker 29 stopped. E0705 21:52:29.297707 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0705 21:52:29.297713 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2rc80blalc4pj5c2hirn6qrh369a4h15-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2rc80blalc4pj5c2hirn6qrh369a4h15-primary-cert-bundle-secret" not found I0705 21:52:29.297767 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=403fe52afec8e153bc4b4d8ed103179948dc7dfae12fc1d3970043e960266fe1 I0705 21:52:29.297793 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 154.257377ms to process 1 records I0705 21:52:29.409605 1 sca.go:266] etc-pki-entitlement secret successfully created I0705 21:52:29.409621 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated W0705 21:52:30.199697 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0705 21:52:31.199863 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0705 21:52:31.627526 1 tasks_processing.go:74] worker 28 stopped. I0705 21:52:31.627695 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=08d99d4491370a8c3ea1978e8b55414287dcf997311f1aa6f707bd4c2668d6c5 I0705 21:52:31.627767 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=b1d3a57201f472fba248e6e22732d03f4ebac77f3d7204455295e9c6f31f376a I0705 21:52:31.627811 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=1a995d423a30cd8b2d276820daf8eec302ceee675edab61e8894e9e00c3e210f I0705 21:52:31.627826 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0705 21:52:31.627860 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=fe16799c8b9a52978dee08cc16b24a47c690e857794cd214fe6c168b9acd8cd4 I0705 21:52:31.627885 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0705 21:52:31.627926 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=1c8c40c0c11bfc69dbe8f67ef2326772a8a4dbb836383567a4f3f6ddb7866559 I0705 21:52:31.627974 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=87a72a02f9987465baa5fef4753030888ad262cc9daab9bf5620abea51e54903 I0705 21:52:31.628011 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=2bef1f23a905033c920ad4930c8d3c53af0c191567b7c88616b58bddf4a2c082 I0705 21:52:31.628062 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=2420e7deeff46a972e513b2d5b26a6e664c19b349f3845bdd1791246cb5bb6dc I0705 21:52:31.628616 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=4346cf98de8e7e4986a9f35a412f90e698a3702147f546271ec6de0ee107cc66 I0705 21:52:31.628654 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=692a0ccf17acd1f758de579a45ba516d57f983cd3b7ee017d163e89d7e7ed8d7 I0705 21:52:31.628665 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0705 21:52:31.628703 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=392907c339e2fd5f64ec8f7b6fd897f242d8476c5e420c2c7f5519b47e2c6e42 I0705 21:52:31.628719 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0705 21:52:31.628746 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=22bb8cfd1be8fa26830030442042efbe8764f735fa142f9877f61e40940e6ee1 I0705 21:52:31.628760 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0705 21:52:31.628796 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=e2c728f113f39e753d449728f97673eccbd39c989fcc8d00c6d52af7c3875b4a I0705 21:52:31.628809 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0705 21:52:31.628845 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=07a08c2e77492ec5518ac380d749e1871c286d652aa68f967462f5a3676f785d I0705 21:52:31.629044 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=f07333f99aca505e39191bed3bbc3c00eb7075e1a5619189aa9f1fdef97b32b0 I0705 21:52:31.629058 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0705 21:52:31.629076 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0705 21:52:31.629111 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0705 21:52:31.629213 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=63adbe934b1b615dbec512ea82fbb1fb685fbf34112c6715ba1fe36cf14f7625 I0705 21:52:31.629257 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=c4854116f40c26f6d1c2c229eb45c7eb9d7743c56332eda08fd683bec7979dfa I0705 21:52:31.629279 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0705 21:52:31.629306 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=889477d58932f0e9f339bdbcc65e83abb42d51c658742ebd33d0dd6afc98ec7e I0705 21:52:31.629320 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0705 21:52:31.629348 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=44441bf3ef0d013d8f3667e0c45735b2cfffdf8d0b02324f97efd7f6ec9232c5 I0705 21:52:31.629364 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0705 21:52:31.629385 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=3dff44a3d911867c9d0237aa06f7d81af936f75f3172ea3c8f5ffa40061009a5 I0705 21:52:31.629407 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=3d933f524d27e3a837669bd1b381b7b3285d6eb58a3711306c3854b5f208862e I0705 21:52:31.629432 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=bf686c36f3d0a44d8646ecf4d2e3425ee8f747a33b7268cb2eac65b902ac7307 I0705 21:52:31.629467 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=745bc9486d9de2cc07d6c75ab108b2e313eb17709e7a9a2ff42bd647d0a6c386 I0705 21:52:31.629480 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0705 21:52:31.629520 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=dbd3a2dc54514b23d2851f593407069094b84bcf25c93ab8ff9bd0e2eb8583d7 I0705 21:52:31.629546 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0705 21:52:31.629558 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0705 21:52:31.629570 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.484255787s to process 39 records W0705 21:52:32.233496 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0705 21:52:33.199480 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0705 21:52:34.199739 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0705 21:52:34.199838 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0705 21:52:34.199886 1 tasks_processing.go:74] worker 4 stopped. E0705 21:52:34.199929 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0705 21:52:34.199986 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0705 21:52:34.200033 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.018271382s to process 1 records I0705 21:52:42.196453 1 tasks_processing.go:74] worker 21 stopped. I0705 21:52:42.196495 1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b I0705 21:52:42.196507 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 13.053388999s to process 1 records I0705 21:52:42.967527 1 tasks_processing.go:74] worker 17 stopped. I0705 21:52:42.967819 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=b5628417db3123c06333720ebefb8c69f4980e1afaf323f1ff9cb97dfe717013 I0705 21:52:42.967837 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.808787072s to process 1 records E0705 21:52:42.967934 1 periodic.go:247] "Unhandled Error" err="clusterconfig failed after 13.825s with: function \"active_alerts\" failed with an error, function \"support_secret\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"tsdb_status\" failed with an error, function \"silenced_alerts\" failed with an error, function \"metrics\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machines\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0705 21:52:42.969035 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "active_alerts" failed with an error, function "support_secret" failed with an error, function "machine_healthchecks" failed with an error, function "tsdb_status" failed with an error, function "silenced_alerts" failed with an error, function "metrics" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machines" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0705 21:52:42.969049 1 periodic.go:209] Running workloads gatherer I0705 21:52:42.969064 1 tasks_processing.go:45] number of workers: 2 I0705 21:52:42.969071 1 tasks_processing.go:69] worker 1 listening for tasks. I0705 21:52:42.969075 1 tasks_processing.go:71] worker 1 working on workload_info task. I0705 21:52:42.969097 1 tasks_processing.go:69] worker 0 listening for tasks. I0705 21:52:42.969111 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0705 21:52:42.978578 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-35.ec2.internal... I0705 21:52:42.978589 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-82.ec2.internal... I0705 21:52:43.010028 1 tasks_processing.go:74] worker 0 stopped. I0705 21:52:43.010047 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 40.900153ms to process 0 records I0705 21:52:49.510893 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 7s I0705 21:52:49.538686 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0705 21:52:49.551060 1 gather_workloads_info.go:387] No image sha256:80748ba08e1c264a8c105e7f607eff386a66378e024443a844993ee9292858c1 (14ms) I0705 21:52:49.562511 1 gather_workloads_info.go:387] No image sha256:04c87c054a3f366a7dfbe0a93ebb0c80a098ee16842c5794b67c1202eec61996 (11ms) I0705 21:52:49.572431 1 gather_workloads_info.go:387] No image sha256:765f0d23b637f685f98a31bd47c131b03cf72a40761a3f9a9d6320faa3c33733 (10ms) I0705 21:52:49.582583 1 gather_workloads_info.go:387] No image sha256:2904a78e2eb73fd6a9bb94c105c2a056831fb4113fbb7b0607c50adc9d879c9b (10ms) I0705 21:52:49.592329 1 gather_workloads_info.go:387] No image sha256:25bda9d34e23ba6bc9df5cf7104bdb237269740a3a76ee025f88b34e68b7e2b5 (10ms) I0705 21:52:49.601859 1 gather_workloads_info.go:387] No image sha256:0092c6ff139f8698c91349d507eb3182bc5d6c4ae040f672e347ff504accdea2 (10ms) I0705 21:52:49.611721 1 gather_workloads_info.go:387] No image sha256:1e65f9f5490fbc930d2dce96f03ddcb28990b839d7dc011ef5de5bec6c5095bf (10ms) I0705 21:52:49.621706 1 gather_workloads_info.go:387] No image sha256:3d6ea9e1633f38070718a1357fcb80e6cb3932aafcc4d65e659795b186c5da15 (10ms) I0705 21:52:49.631351 1 gather_workloads_info.go:387] No image sha256:7adc1eab05d6724c76ba751f6df816b08d6e70b78dee9eb94fa6fd9690542c98 (10ms) I0705 21:52:49.640943 1 gather_workloads_info.go:387] No image sha256:b99613494e9d6a4a8207323872d63cc02b830fc80a8bf972e62c64381aa4cd17 (10ms) I0705 21:52:49.650820 1 gather_workloads_info.go:387] No image sha256:a56211d075aa43cbb491f669a5b2e46ee023dc95b7d51dbac28f463948c5ad61 (10ms) I0705 21:52:49.748876 1 gather_workloads_info.go:387] No image sha256:2598489729a4b258e4ecda4a06f6875133f2a10ced5c5241f8a57a8a05418e36 (98ms) I0705 21:52:49.848749 1 gather_workloads_info.go:387] No image sha256:fab570ecdc9c3620b032bc4c5b44240f5ebac0bb9e48dfeea32bcd36eb462e5a (100ms) I0705 21:52:49.948849 1 gather_workloads_info.go:387] No image sha256:a498046d64605bcccee2440aa4f04a4602baaae263cf01d977ec5208e876b1fd (100ms) I0705 21:52:50.049131 1 gather_workloads_info.go:387] No image sha256:56a85660a445eced5c79a595a0eccf590087c5672d50f49d4c25ad52f9a44f04 (100ms) I0705 21:52:50.148718 1 gather_workloads_info.go:387] No image sha256:ca1344cb64140188b7cae7bbc51fb751566c0b0c97d5e39b5850e628032c4a5e (100ms) I0705 21:52:50.249646 1 gather_workloads_info.go:387] No image sha256:d3209a996e5ce362136e70a0bcf06d89973905de73e234b644d8ae8886d067f1 (101ms) I0705 21:52:50.348842 1 gather_workloads_info.go:387] No image sha256:95418d0ba6fc0aa96647d97cf6530967f7d7ae97a82ce7f7e41c8e44310b0b72 (99ms) I0705 21:52:50.449380 1 gather_workloads_info.go:387] No image sha256:6e5df2aab9ce759e8f957341a11c087c233307e3c8f8760d21270efc19ccd3ce (101ms) I0705 21:52:50.548422 1 gather_workloads_info.go:387] No image sha256:f1f19d18c94f996546a20ce31a68d177d0fc13378e5452b9ff8dfca9d467aa95 (99ms) I0705 21:52:50.648885 1 gather_workloads_info.go:387] No image sha256:513b1166e376feca472fdd5174f6abf0abf02925d6a21b7b4d89a2ff2fc67bed (100ms) I0705 21:52:50.750103 1 gather_workloads_info.go:387] No image sha256:943018739e3db1763c3184b460dbc409e058abbac76d57b9927faad317be85e4 (101ms) I0705 21:52:50.849564 1 gather_workloads_info.go:387] No image sha256:3612c5a433effd17b7aff7ff6edf4fb27b61153908e3a0294158e4886a1d600d (99ms) I0705 21:52:50.947833 1 gather_workloads_info.go:387] No image sha256:2e564f336c77116053f34d4201d364d8da04e789cfffa0ea422574c95f2d6404 (98ms) I0705 21:52:51.088147 1 gather_workloads_info.go:387] No image sha256:2e57e192c3c1240fd935dcd55c8fde5e70e78bf81d6176c96edf21fafe59f8ba (140ms) I0705 21:52:51.156010 1 gather_workloads_info.go:387] No image sha256:5a95c19d82767e0235b4edb4a0536482c816904897aae1dc3eb255cb52b87a9f (68ms) I0705 21:52:51.248917 1 gather_workloads_info.go:387] No image sha256:1477f3287cc75352305274f248f781c1b7c2a8696f4f7bc450c45a4a5f5b10ea (93ms) I0705 21:52:51.349156 1 gather_workloads_info.go:387] No image sha256:03cf4cd7ef1518610c6c7b3ad27d1622d82e98e3dc6e3f8e5d0fceb5c8d3786e (100ms) I0705 21:52:51.448930 1 gather_workloads_info.go:387] No image sha256:fac526ab9335a9a8decb028a30673f4ae718deae8da114f36103b6438ba6458f (100ms) I0705 21:52:51.548922 1 gather_workloads_info.go:387] No image sha256:50197f22710766515f67944a779e00dd9ae3d17b18732d7324a970353b11f292 (100ms) I0705 21:52:51.647984 1 gather_workloads_info.go:387] No image sha256:7e7e39f59a298ff2ca6aba00ce861866dc16bf39398e78523ba4e7600c46dea9 (99ms) I0705 21:52:51.754333 1 gather_workloads_info.go:398] Found image sha256:32d4561d4bf1b3c4e3a7878f918f9c1422bab85bdf8711fbdf905d59692fee81 (106ms) I0705 21:52:51.848195 1 gather_workloads_info.go:387] No image sha256:90bdf9e678fe18e35401e34e1cb0912d822e9cb32b9e41cc545870540e9a87f2 (94ms) I0705 21:52:51.948889 1 gather_workloads_info.go:387] No image sha256:0af73ad7a47c9ff53269ae2bc5163bf9e389085220c34e4bd42c92c4e7a85f6d (101ms) I0705 21:52:52.049681 1 gather_workloads_info.go:387] No image sha256:09e27b3a67ebeda04d3e597e0561bfb40550de78bdecb53f2f9ff3dfb1cfe7ea (101ms) I0705 21:52:52.149891 1 gather_workloads_info.go:387] No image sha256:91828234f107c068c8a4966d08370ae7b73e637651dbc6d92c18c4553402c22c (100ms) I0705 21:52:52.251081 1 gather_workloads_info.go:387] No image sha256:a258c226562adb14e3a163a1940938526ee6a0928982a7667d85d9a7334ce639 (101ms) I0705 21:52:52.348855 1 gather_workloads_info.go:387] No image sha256:101993f6e65d6fda8b90c583b718b720809182961f69bc777295f47e6df4d7ce (98ms) I0705 21:52:52.451459 1 gather_workloads_info.go:387] No image sha256:289816958633a763a72dbc44e1dad40466223164e7e253039514f0d974ea5d21 (103ms) I0705 21:52:52.548029 1 gather_workloads_info.go:387] No image sha256:695cf2f0cc07683c2a3ce1eaf3e56fe18abc6e2bac716f7d9843f5d173b9df52 (97ms) I0705 21:52:52.649406 1 gather_workloads_info.go:387] No image sha256:5f0b67cfbbc381243fb91ccc17345b56d05f4d717c667e8c644e5bf05633ba71 (101ms) I0705 21:52:52.762490 1 gather_workloads_info.go:387] No image sha256:521712486e2c6e3c020dad6a1cb340db8e55665b69f7c208fab9cd9e965fd588 (113ms) I0705 21:52:52.857211 1 gather_workloads_info.go:387] No image sha256:653c666f842c13e0baae2e89a9b1efe0e2ef56f621ffb5b32005115d2a26ab8c (95ms) I0705 21:52:52.966235 1 gather_workloads_info.go:387] No image sha256:36b9e89c3cfcf1ab9ae500486e38afb6862cba48cb0b4d84a09508ab8f3d299f (109ms) I0705 21:52:53.049505 1 gather_workloads_info.go:387] No image sha256:74713675d580a47a56b628469c4c05c41e114e0b935960d718ff0cf7e72f613d (83ms) I0705 21:52:53.147965 1 gather_workloads_info.go:387] No image sha256:4556896f77307821531ef91b7b7faccb82b824ea695693b2989f597f0deca038 (98ms) I0705 21:52:53.248526 1 gather_workloads_info.go:398] Found image sha256:b3909bf664c77097f75b3768830863d642eed3815dab2bfb4415c771ca2d5007 (101ms) I0705 21:52:53.349383 1 gather_workloads_info.go:387] No image sha256:a505dd5b62d744c2cfda2666b5e21e0a4704354a3114488bd3291456085e1e33 (101ms) I0705 21:52:53.448849 1 gather_workloads_info.go:387] No image sha256:0a99240166165eb5718e7516a43282fe32df9c7c5e809b31b58abe44e42ff94d (99ms) I0705 21:52:53.548857 1 gather_workloads_info.go:387] No image sha256:a0105d1eb62cf6ac9e5e2ef28d3e89bf6dc514bc594fc7090fe5a5ee18a09c87 (100ms) I0705 21:52:53.648473 1 gather_workloads_info.go:387] No image sha256:c940ea87e7d133d75ba0002ef00c0806825eed3db8094cdb260d1bac18127733 (100ms) I0705 21:52:53.748290 1 gather_workloads_info.go:387] No image sha256:47154813651033d59751fb655a384dbffb64dd26f10bd7f3be0c3128d0486356 (100ms) I0705 21:52:53.849508 1 gather_workloads_info.go:387] No image sha256:ef2bfb546cd8aac31b7671585f718f44b268ed43a9c8562016ab741558a7dfa1 (101ms) I0705 21:52:53.947997 1 gather_workloads_info.go:387] No image sha256:7bd3e88d6d3f878a89203df590b684c5a77bf08df4717451d3197ab74683a80e (98ms) I0705 21:52:54.048725 1 gather_workloads_info.go:387] No image sha256:1a2532940843248c57d52141185dd71fbc393ab28b65d48f682038632c1dbbad (101ms) I0705 21:52:54.149484 1 gather_workloads_info.go:387] No image sha256:76cc47f3775560477ec548c7b6fddbd99112ad6604f790697eef546576051fb7 (101ms) I0705 21:52:54.247902 1 gather_workloads_info.go:387] No image sha256:7b31223098f08328f5ddea8e5b871dbbd5f5a61ec550e8956f66793c0c6031a9 (98ms) I0705 21:52:54.247938 1 tasks_processing.go:74] worker 1 stopped. E0705 21:52:54.247948 1 gather.go:140] gatherer "workloads" function "workload_info" failed with the error: received unexpected status code 500 Internal Server Error from https://10.129.0.13:8443/gather_runtime_info I0705 21:52:54.248289 1 recorder.go:75] Recording config/workload_info with fingerprint=61f9c8ab01bf819bc8074366973340cdc67e26a718f2fa50c2c4b871386cd6f7 I0705 21:52:54.248307 1 gather.go:177] gatherer "workloads" function "workload_info" took 11.278852715s to process 1 records E0705 21:52:54.248335 1 periodic.go:247] "Unhandled Error" err="workloads failed after 11.279s with: function \"workload_info\" failed with an error" I0705 21:52:54.249433 1 controllerstatus.go:89] name=periodic-workloads healthy=false reason=PeriodicGatherFailed message=Source workloads could not be retrieved: function "workload_info" failed with an error I0705 21:52:54.249444 1 periodic.go:209] Running conditional gatherer I0705 21:52:54.257761 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules I0705 21:52:54.361488 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer E0705 21:52:54.363830 1 conditional_gatherer.go:324] unable to update alerts cache: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?match%5B%5D=ALERTS%7Balertstate%3D%22firing%22%7D&query=ALERTS": dial tcp 172.30.140.139:9091: connect: connection refused I0705 21:52:54.363902 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0705 21:52:54.372011 1 conditional_gatherer.go:392] cluster version is '4.20.8' E0705 21:52:54.372025 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0705 21:52:54.372030 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0705 21:52:54.372033 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0705 21:52:54.372036 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0705 21:52:54.372039 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0705 21:52:54.372042 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0705 21:52:54.372045 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0705 21:52:54.372048 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0705 21:52:54.372051 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0705 21:52:54.372054 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0705 21:52:54.372068 1 tasks_processing.go:45] number of workers: 3 I0705 21:52:54.372082 1 tasks_processing.go:69] worker 2 listening for tasks. I0705 21:52:54.372088 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0705 21:52:54.372091 1 tasks_processing.go:69] worker 0 listening for tasks. I0705 21:52:54.372100 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0705 21:52:54.372106 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0705 21:52:54.372105 1 tasks_processing.go:69] worker 1 listening for tasks. I0705 21:52:54.372114 1 tasks_processing.go:74] worker 1 stopped. I0705 21:52:54.372220 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=9fafdcbf373f1f367f1d69551ea59114592cce1681189be0f3dde774e0530661 I0705 21:52:54.372238 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 917ns to process 1 records I0705 21:52:54.372289 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0705 21:52:54.372301 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.164µs to process 1 records I0705 21:52:54.372309 1 tasks_processing.go:74] worker 0 stopped. I0705 21:52:54.372418 1 tasks_processing.go:74] worker 2 stopped. I0705 21:52:54.372428 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 306.916µs to process 0 records I0705 21:52:54.372441 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0705 21:52:54.372457 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0705 21:52:54.401683 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=2680bab4d8b2e3b6d9d08a5e18fa7b0c7276a2526e97d7c6162425497c2c2d0b I0705 21:52:54.401814 1 diskrecorder.go:70] Writing 103 records to /var/lib/insights-operator/insights-2026-07-05-215254.tar.gz I0705 21:52:54.410626 1 diskrecorder.go:51] Wrote 103 records to disk in 8ms I0705 21:52:54.410652 1 periodic.go:278] Gathering cluster info every 2h0m0s I0705 21:52:54.410666 1 periodic.go:279] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0705 21:53:59.143213 1 diskrecorder.go:223] Found files to send: insights-2026-07-05-215254.tar.gz I0705 21:53:59.143244 1 insightsuploader.go:150] Checking archives to upload periodically every 15m24.136459472s I0705 21:53:59.143254 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0705 21:53:59.155327 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0705 21:53:59.341381 1 requests.go:87] Successfully reported id=2026-07-05T21:53:59Z x-rh-insights-request-id=774ac76bf8764993ad9323e7679c8915, wrote=48615 I0705 21:53:59.341438 1 insightsuploader.go:187] Uploaded report successfully in 198.175401ms I0705 21:53:59.341461 1 controller.go:128] Initializing last reported time to 2026-07-05T21:53:59Z I0705 21:53:59.341478 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0705 21:53:59.341486 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0705 21:53:59.341494 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0705 21:53:59.349081 1 controller.go:512] The operator is healthy I0705 21:54:29.164822 1 controller.go:512] The operator is healthy I0705 21:55:04.696675 1 insightsreport.go:137] Pulling report from smart-proxy I0705 21:55:04.696699 1 insightsreport.go:149] Retrieving report I0705 21:55:04.706027 1 requests.go:110] Retrieving report for cluster: 37dee443-ef17-414a-b411-6bbd0783b179 I0705 21:55:04.706040 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/37dee443-ef17-414a-b411-6bbd0783b179/reports I0705 21:55:04.708914 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/37dee443-ef17-414a-b411-6bbd0783b179/reports I0705 21:55:04.934354 1 insightsreport.go:184] Report retrieved I0705 21:55:04.983448 1 insightsreport.go:239] Report retrieved correctly I0705 21:56:29.164381 1 controller.go:512] The operator is healthy I0705 21:57:29.136029 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0705 21:57:29.145657 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0705 21:57:29.145681 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0705 21:57:29.154409 1 secretconfigobserver.go:119] support secret does not exist I0705 21:58:29.179438 1 controller.go:512] The operator is healthy I0705 22:00:29.164445 1 controller.go:512] The operator is healthy I0705 22:02:29.131616 1 configmapobserver.go:84] configmaps "insights-config" not found I0705 22:02:29.137443 1 configmapobserver.go:84] configmaps "insights-config" not found I0705 22:02:29.154488 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0705 22:02:29.160652 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0705 22:02:29.160670 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0705 22:02:29.163598 1 controller.go:512] The operator is healthy I0705 22:02:29.166160 1 secretconfigobserver.go:119] support secret does not exist I0705 22:04:29.172286 1 controller.go:512] The operator is healthy I0705 22:06:29.166148 1 controller.go:512] The operator is healthy I0705 22:07:29.167198 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0705 22:07:29.177738 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0705 22:07:29.177758 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0705 22:07:29.184227 1 secretconfigobserver.go:119] support secret does not exist I0705 22:08:29.164889 1 controller.go:512] The operator is healthy I0705 22:09:28.479123 1 insightsuploader.go:146] Nothing to report since 2026-07-05T21:53:59Z I0705 22:10:29.165141 1 controller.go:512] The operator is healthy I0705 22:12:29.130288 1 configmapobserver.go:84] configmaps "insights-config" not found I0705 22:12:29.163939 1 controller.go:512] The operator is healthy I0705 22:12:29.184805 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0705 22:12:29.191700 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0705 22:12:29.191718 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0705 22:12:29.197999 1 secretconfigobserver.go:119] support secret does not exist I0705 22:14:29.164606 1 controller.go:512] The operator is healthy I0705 22:16:29.165891 1 controller.go:512] The operator is healthy I0705 22:17:29.198487 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0705 22:17:29.208519 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0705 22:17:29.208537 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0705 22:17:29.214747 1 secretconfigobserver.go:119] support secret does not exist I0705 22:18:29.164216 1 controller.go:512] The operator is healthy I0705 22:20:29.169746 1 controller.go:512] The operator is healthy I0705 22:22:29.130740 1 configmapobserver.go:84] configmaps "insights-config" not found I0705 22:22:29.143818 1 configmapobserver.go:84] configmaps "insights-config" not found I0705 22:22:29.164066 1 controller.go:512] The operator is healthy I0705 22:22:29.215203 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0705 22:22:29.225001 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0705 22:22:29.225073 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0705 22:22:29.231602 1 secretconfigobserver.go:119] support secret does not exist I0705 22:24:29.163924 1 controller.go:512] The operator is healthy I0705 22:25:34.890613 1 insightsuploader.go:146] Nothing to report since 2026-07-05T21:53:59Z I0705 22:26:29.164554 1 controller.go:512] The operator is healthy I0705 22:27:29.232017 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0705 22:27:29.239921 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0705 22:27:29.239950 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0705 22:27:29.246383 1 secretconfigobserver.go:119] support secret does not exist I0705 22:28:29.164557 1 controller.go:512] The operator is healthy I0705 22:30:29.164915 1 controller.go:512] The operator is healthy I0705 22:32:29.134466 1 configmapobserver.go:84] configmaps "insights-config" not found I0705 22:32:29.164034 1 controller.go:512] The operator is healthy I0705 22:32:29.247211 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0705 22:32:29.253737 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0705 22:32:29.253755 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0705 22:32:29.260002 1 secretconfigobserver.go:119] support secret does not exist I0705 22:34:29.163978 1 controller.go:512] The operator is healthy I0705 22:36:29.165354 1 controller.go:512] The operator is healthy I0705 22:37:29.261041 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0705 22:37:29.267787 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0705 22:37:29.267806 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0705 22:37:29.273615 1 secretconfigobserver.go:119] support secret does not exist I0705 22:38:29.164606 1 controller.go:512] The operator is healthy I0705 22:40:29.165564 1 controller.go:512] The operator is healthy I0705 22:42:05.838342 1 insightsuploader.go:146] Nothing to report since 2026-07-05T21:53:59Z I0705 22:42:29.136882 1 configmapobserver.go:84] configmaps "insights-config" not found I0705 22:42:29.142975 1 configmapobserver.go:84] configmaps "insights-config" not found I0705 22:42:29.164543 1 controller.go:512] The operator is healthy I0705 22:42:29.274666 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0705 22:42:29.281284 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0705 22:42:29.281303 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0705 22:42:29.287062 1 secretconfigobserver.go:119] support secret does not exist I0705 22:44:29.164450 1 controller.go:512] The operator is healthy I0705 22:46:29.164567 1 controller.go:512] The operator is healthy