I0703 18:02:31.761371 1 cmd.go:253] Using service-serving-cert provided certificates I0703 18:02:31.761909 1 observer_polling.go:159] Starting file observer I0703 18:02:32.480878 1 operator.go:60] Starting insights-operator v0.0.0-master+$Format:%H$ I0703 18:02:32.481084 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0703 18:02:32.481659 1 secure_serving.go:57] Forcing use of http/1.1 only W0703 18:02:32.481679 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. I0703 18:02:32.481674 1 simple_featuregate_reader.go:171] Starting feature-gate-detector W0703 18:02:32.481685 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0703 18:02:32.481718 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0703 18:02:32.481724 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0703 18:02:32.481728 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0703 18:02:32.481732 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0703 18:02:32.486589 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"481e9154-1b3a-44db-8c7a-8c37df97486e", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}} I0703 18:02:32.486580 1 operator.go:125] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GatewayAPI GatewayAPIController HighlyAvailableArbiter ImageVolume IngressControllerLBSubnetsAWS KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController SigstoreImageVerification StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPClusterHostedDNSInstall GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall ImageModeStatusReporting ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesAzure ManagedBootImagesvSphere MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutatingAdmissionPolicy NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterOperations NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PreconfiguredUDNAddresses SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeAttributesClass VolumeGroupSnapshot] I0703 18:02:32.487187 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0703 18:02:32.487203 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0703 18:02:32.487263 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0703 18:02:32.487337 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0703 18:02:32.487314 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0703 18:02:32.487365 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0703 18:02:32.487687 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0703 18:02:32.488006 1 secure_serving.go:213] Serving securely on [::]:8443 I0703 18:02:32.488057 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0703 18:02:32.492249 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0703 18:02:32.492272 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0703 18:02:32.492369 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0703 18:02:32.497231 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0703 18:02:32.497250 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0703 18:02:32.500621 1 secretconfigobserver.go:119] support secret does not exist I0703 18:02:32.504239 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0703 18:02:32.508030 1 secretconfigobserver.go:119] support secret does not exist I0703 18:02:32.510757 1 recorder.go:161] Pruning old reports every 4h15m33s, max age is 288h0m0s I0703 18:02:32.514869 1 periodic.go:209] Running clusterconfig gatherer I0703 18:02:32.514988 1 tasks_processing.go:45] number of workers: 32 I0703 18:02:32.515004 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0703 18:02:32.515016 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0703 18:02:32.515050 1 tasks_processing.go:69] worker 31 listening for tasks. I0703 18:02:32.515062 1 tasks_processing.go:71] worker 31 working on pod_network_connectivity_checks task. I0703 18:02:32.515066 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0703 18:02:32.515072 1 insightsreport.go:296] Starting report retriever I0703 18:02:32.515078 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0703 18:02:32.515092 1 tasks_processing.go:69] worker 15 listening for tasks. I0703 18:02:32.515163 1 tasks_processing.go:69] worker 0 listening for tasks. I0703 18:02:32.515194 1 tasks_processing.go:69] worker 1 listening for tasks. I0703 18:02:32.515237 1 tasks_processing.go:69] worker 2 listening for tasks. I0703 18:02:32.515271 1 tasks_processing.go:69] worker 3 listening for tasks. I0703 18:02:32.515314 1 tasks_processing.go:69] worker 4 listening for tasks. I0703 18:02:32.515342 1 tasks_processing.go:69] worker 5 listening for tasks. I0703 18:02:32.515368 1 tasks_processing.go:69] worker 6 listening for tasks. I0703 18:02:32.515389 1 tasks_processing.go:69] worker 16 listening for tasks. I0703 18:02:32.515399 1 tasks_processing.go:69] worker 17 listening for tasks. I0703 18:02:32.515409 1 tasks_processing.go:69] worker 18 listening for tasks. I0703 18:02:32.515418 1 tasks_processing.go:69] worker 19 listening for tasks. I0703 18:02:32.515499 1 tasks_processing.go:69] worker 28 listening for tasks. I0703 18:02:32.515426 1 tasks_processing.go:69] worker 20 listening for tasks. I0703 18:02:32.515442 1 tasks_processing.go:69] worker 21 listening for tasks. I0703 18:02:32.515452 1 tasks_processing.go:69] worker 22 listening for tasks. I0703 18:02:32.515459 1 tasks_processing.go:69] worker 23 listening for tasks. I0703 18:02:32.515467 1 tasks_processing.go:69] worker 24 listening for tasks. I0703 18:02:32.515474 1 tasks_processing.go:69] worker 25 listening for tasks. I0703 18:02:32.515483 1 tasks_processing.go:69] worker 26 listening for tasks. I0703 18:02:32.515490 1 tasks_processing.go:69] worker 27 listening for tasks. I0703 18:02:32.515543 1 tasks_processing.go:69] worker 29 listening for tasks. I0703 18:02:32.515553 1 tasks_processing.go:69] worker 30 listening for tasks. I0703 18:02:32.515570 1 tasks_processing.go:71] worker 30 working on crds task. I0703 18:02:32.515605 1 tasks_processing.go:69] worker 7 listening for tasks. I0703 18:02:32.515632 1 tasks_processing.go:71] worker 7 working on storage_cluster task. I0703 18:02:32.515898 1 tasks_processing.go:71] worker 15 working on qemu_kubevirt_launcher_logs task. I0703 18:02:32.516089 1 tasks_processing.go:69] worker 8 listening for tasks. I0703 18:02:32.516142 1 tasks_processing.go:69] worker 9 listening for tasks. I0703 18:02:32.516163 1 tasks_processing.go:71] worker 19 working on support_secret task. I0703 18:02:32.516184 1 tasks_processing.go:69] worker 10 listening for tasks. I0703 18:02:32.516189 1 tasks_processing.go:71] worker 20 working on sap_config task. I0703 18:02:32.516197 1 tasks_processing.go:69] worker 11 listening for tasks. I0703 18:02:32.516206 1 tasks_processing.go:71] worker 26 working on cluster_apiserver task. I0703 18:02:32.516214 1 tasks_processing.go:71] worker 24 working on openstack_controlplanes task. I0703 18:02:32.516224 1 tasks_processing.go:71] worker 25 working on active_alerts task. I0703 18:02:32.516184 1 tasks_processing.go:71] worker 28 working on nodes task. I0703 18:02:32.516339 1 tasks_processing.go:71] worker 27 working on dvo_metrics task. I0703 18:02:32.516170 1 tasks_processing.go:69] worker 12 listening for tasks. I0703 18:02:32.516375 1 tasks_processing.go:71] worker 0 working on schedulers task. I0703 18:02:32.516177 1 tasks_processing.go:71] worker 23 working on overlapping_namespace_uids task. I0703 18:02:32.516473 1 tasks_processing.go:71] worker 5 working on config_maps task. I0703 18:02:32.516450 1 tasks_processing.go:71] worker 1 working on storage_classes task. I0703 18:02:32.516550 1 tasks_processing.go:71] worker 17 working on infrastructures task. I0703 18:02:32.516454 1 tasks_processing.go:71] worker 2 working on monitoring_persistent_volumes task. I0703 18:02:32.516452 1 tasks_processing.go:71] worker 29 working on pdbs task. I0703 18:02:32.516465 1 tasks_processing.go:71] worker 3 working on silenced_alerts task. I0703 18:02:32.516469 1 tasks_processing.go:71] worker 4 working on node_logs task. I0703 18:02:32.516535 1 tasks_processing.go:71] worker 6 working on feature_gates task. I0703 18:02:32.516540 1 tasks_processing.go:71] worker 16 working on authentication task. I0703 18:02:32.517456 1 tasks_processing.go:71] worker 21 working on ingress_certificates task. I0703 18:02:32.517602 1 tasks_processing.go:71] worker 22 working on openstack_dataplanenodesets task. I0703 18:02:32.516686 1 tasks_processing.go:71] worker 18 working on container_runtime_configs task. I0703 18:02:32.518010 1 tasks_processing.go:69] worker 14 listening for tasks. I0703 18:02:32.518022 1 tasks_processing.go:69] worker 13 listening for tasks. I0703 18:02:32.518029 1 tasks_processing.go:71] worker 14 working on operators task. I0703 18:02:32.518032 1 tasks_processing.go:71] worker 13 working on image_pruners task. I0703 18:02:32.518088 1 tasks_processing.go:71] worker 8 working on container_images task. I0703 18:02:32.518204 1 tasks_processing.go:71] worker 10 working on certificate_signing_requests task. I0703 18:02:32.518251 1 tasks_processing.go:71] worker 9 working on operators_pods_and_events task. I0703 18:02:32.518260 1 tasks_processing.go:71] worker 11 working on machine_sets task. E0703 18:02:32.518893 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0703 18:02:32.518958 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 3.80505ms to process 0 records I0703 18:02:32.518266 1 tasks_processing.go:71] worker 12 working on nodenetworkstates task. I0703 18:02:32.518937 1 tasks_processing.go:71] worker 31 working on nodenetworkconfigurationpolicies task. I0703 18:02:32.520172 1 tasks_processing.go:71] worker 7 working on jaegers task. I0703 18:02:32.520185 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 4.507609ms to process 0 records I0703 18:02:32.523928 1 tasks_processing.go:71] worker 24 working on olm_operators task. I0703 18:02:32.523936 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 7.698043ms to process 0 records I0703 18:02:32.524041 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 7.804294ms to process 0 records I0703 18:02:32.524093 1 controller.go:128] Initializing last reported time to 0001-01-01T00:00:00Z I0703 18:02:32.524110 1 tasks_processing.go:71] worker 20 working on clusterroles task. I0703 18:02:32.524115 1 controller.go:326] The initial operator extension status is healthy I0703 18:02:32.524126 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0703 18:02:32.524131 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0703 18:02:32.524136 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0703 18:02:32.524158 1 controller.go:489] The operator is still being initialized I0703 18:02:32.524169 1 controller.go:512] The operator is healthy I0703 18:02:32.524302 1 recorder.go:75] Recording config/apiserver with fingerprint=f0813bb252341a1dd7453b19b067543825d362c0576070149b37bbeaebee2876 I0703 18:02:32.524348 1 tasks_processing.go:71] worker 26 working on oauths task. I0703 18:02:32.524359 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 7.830586ms to process 1 records I0703 18:02:32.524300 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s W0703 18:02:32.524395 1 operator.go:288] started I0703 18:02:32.524303 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s I0703 18:02:32.524418 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0703 18:02:32.529985 1 tasks_processing.go:71] worker 18 working on openstack_dataplanedeployments task. I0703 18:02:32.530006 1 gather_logs.go:145] no pods in namespace were found I0703 18:02:32.530017 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 12.170979ms to process 0 records I0703 18:02:32.530026 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 14.100224ms to process 0 records I0703 18:02:32.530034 1 tasks_processing.go:71] worker 15 working on sap_pods task. I0703 18:02:32.530239 1 tasks_processing.go:71] worker 0 working on machine_config_pools task. I0703 18:02:32.530331 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=08309ab462d05b99f117539d9ff3d0bb205d5665ef0489c4e9c526d3ccc2d529 I0703 18:02:32.530350 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 13.848345ms to process 1 records I0703 18:02:32.530456 1 tasks_processing.go:71] worker 17 working on openshift_machine_api_events task. I0703 18:02:32.530895 1 recorder.go:75] Recording config/infrastructure with fingerprint=f1ead9173cf8d9d00f86149680bc1e0f50ef74b4c3157f1f5a69d68268c6469f I0703 18:02:32.530907 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 13.732147ms to process 1 records I0703 18:02:32.530917 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 10.494896ms to process 0 records I0703 18:02:32.530922 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 12.240181ms to process 0 records I0703 18:02:32.530928 1 tasks_processing.go:71] worker 11 working on machines task. I0703 18:02:32.531052 1 tasks_processing.go:71] worker 7 working on openstack_version task. I0703 18:02:32.536745 1 tasks_processing.go:71] worker 12 working on version task. I0703 18:02:32.536767 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 17.76285ms to process 0 records I0703 18:02:32.537472 1 tasks_processing.go:71] worker 22 working on machine_healthchecks task. I0703 18:02:32.537506 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 19.849878ms to process 0 records I0703 18:02:32.537566 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 18.375977ms to process 0 records I0703 18:02:32.537644 1 tasks_processing.go:71] worker 31 working on sap_datahubs task. I0703 18:02:32.537720 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=71f081c42c26b11fe85ed28875c39a8b56db28ac3f6fa0f54498ae73d1ef6ad5 I0703 18:02:32.537780 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=bda2bb38e70d9b15d8ca435e9ce6d469a9feff66c219ad30d300078300585f09 I0703 18:02:32.537809 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 20.94427ms to process 2 records I0703 18:02:32.537813 1 tasks_processing.go:71] worker 1 working on ingress task. I0703 18:02:32.537933 1 tasks_processing.go:71] worker 6 working on lokistack task. I0703 18:02:32.537961 1 recorder.go:75] Recording config/featuregate with fingerprint=7d8bb0ab3c267b6c9dd382de773651f414ca12390cf7ab225f739a3ed1c4e7e0 I0703 18:02:32.537970 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 20.704672ms to process 1 records I0703 18:02:32.539909 1 tasks_processing.go:71] worker 10 working on service_accounts task. I0703 18:02:32.539924 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 21.659864ms to process 0 records W0703 18:02:32.540369 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0703 18:02:32.549430 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 32.440445ms to process 0 records I0703 18:02:32.549436 1 tasks_processing.go:71] worker 4 working on validating_webhook_configurations task. I0703 18:02:32.549610 1 tasks_processing.go:71] worker 19 working on image_registries task. E0703 18:02:32.549616 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0703 18:02:32.549906 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 33.421479ms to process 0 records I0703 18:02:32.549923 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 33.169523ms to process 0 records I0703 18:02:32.549932 1 tasks_processing.go:71] worker 2 working on machine_configs task. I0703 18:02:32.550218 1 tasks_processing.go:71] worker 13 working on image task. I0703 18:02:32.550573 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=71238e2f71b54c2981ccf16f1bd8e36ab553ea996f394884aa9de90149924848 I0703 18:02:32.550622 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 32.177793ms to process 1 records I0703 18:02:32.550632 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 26.516717ms to process 0 records I0703 18:02:32.550640 1 tasks_processing.go:71] worker 24 working on tsdb_status task. I0703 18:02:32.551404 1 tasks_processing.go:71] worker 15 working on proxies task. I0703 18:02:32.551419 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 21.361356ms to process 0 records I0703 18:02:32.552489 1 tasks_processing.go:71] worker 26 working on machine_autoscalers task. I0703 18:02:32.553346 1 recorder.go:75] Recording config/oauth with fingerprint=141ccf78a3baec872202fcf1a4ae18ab5df838d837f724eeeb529091281e52b8 I0703 18:02:32.553786 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 28.122119ms to process 1 records I0703 18:02:32.554442 1 recorder.go:75] Recording config/node/ip-10-0-0-113.ec2.internal with fingerprint=d031d338431c0cc19d50a7b966547e81e4abd7d2327c8517b683c7d219f1736a I0703 18:02:32.554631 1 recorder.go:75] Recording config/node/ip-10-0-1-109.ec2.internal with fingerprint=d142df0436148fddfc7c1acaf9d93c81d5f42ad2dbaf96cd69460be82445ec3b I0703 18:02:32.554800 1 recorder.go:75] Recording config/node/ip-10-0-2-165.ec2.internal with fingerprint=fa59abfc2cdaf646e653df0362eeba5ef9143fd5039aa9fac74d7f1bb3b271e4 I0703 18:02:32.554840 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 36.621091ms to process 3 records I0703 18:02:32.554876 1 tasks_processing.go:71] worker 28 working on metrics task. I0703 18:02:32.555221 1 tasks_processing.go:71] worker 11 working on aggregated_monitoring_cr_names task. E0703 18:02:32.554877 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0703 18:02:32.555490 1 gather.go:177] gatherer "clusterconfig" function "machines" took 22.531277ms to process 0 records I0703 18:02:32.555804 1 tasks_processing.go:71] worker 16 working on cost_management_metrics_configs task. I0703 18:02:32.556176 1 recorder.go:75] Recording config/authentication with fingerprint=f35f7919f2ffed35a71c92aefe186310ec52973cafd74a8fe7ffcf8d56b60989 I0703 18:02:32.556206 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 36.656518ms to process 1 records I0703 18:02:32.556215 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 23.863181ms to process 0 records I0703 18:02:32.556231 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 23.288888ms to process 0 records I0703 18:02:32.556237 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 24.598759ms to process 0 records I0703 18:02:32.556273 1 controller.go:212] Source scaController *sca.Controller is not ready I0703 18:02:32.556286 1 controller.go:212] Source clusterTransferController *clustertransfer.Controller is not ready I0703 18:02:32.556292 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0703 18:02:32.556297 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0703 18:02:32.556302 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0703 18:02:32.556348 1 controller.go:489] The operator is still being initialized I0703 18:02:32.556398 1 controller.go:512] The operator is healthy I0703 18:02:32.556371 1 tasks_processing.go:71] worker 7 working on mutating_webhook_configurations task. I0703 18:02:32.556498 1 tasks_processing.go:71] worker 29 working on openshift_logging task. E0703 18:02:32.554839 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.134.146:9091: connect: connection refused I0703 18:02:32.556377 1 tasks_processing.go:71] worker 0 working on networks task. I0703 18:02:32.556384 1 tasks_processing.go:71] worker 17 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0703 18:02:32.556875 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=b35fcac60cdd84f7fa76ab4162dfc9014d9cb42eb560f62a857a1a8df2a1d049 I0703 18:02:32.558098 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=9574ef30132cb44564338f0906b80f6789918999807a4e93d9c9b9b463768d25 I0703 18:02:32.558153 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=91eaf266130d1afde210684be6256084e6796da001eee118d96017d5bef57964 I0703 18:02:32.558226 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=4dff82f410e8255fbb147676c8841072fbe1c5e3fd59b3955e2659a4c6dce3dc I0703 18:02:32.558290 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=926e2b5917f53acc04ac5bb2740463e4dbbcefac3d972521842545a10d25c427 I0703 18:02:32.558338 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=536c3166b9492f62608f73b8d5bb6da0578c7a68f710994aae2a87ee7dd3c978 I0703 18:02:32.558387 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=c6af9d942a04e069ba1fdb276feb9d1de39dfb87f94a869f2c86636d7919fbf5 I0703 18:02:32.559150 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=8191088885d4354cfe6ff4db9e0bfdaaa301cd8201729613c30e6dd00829b18c I0703 18:02:32.559227 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=912d05560cc0d08db03eebb643941620fa2f6110f778e0286f0ec63a22aaa18d I0703 18:02:32.559278 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=3f6e08fbf560e197683a2da723f11aae146cd0c4a73123a19cd47e74fbdbe4da I0703 18:02:32.559341 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=4ddd6b57074d18f8828818bcd581dc247d2e49a11126c04ef3ce8126994f5b23 I0703 18:02:32.559356 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 39.633275ms to process 11 records E0703 18:02:32.559369 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.134.146:9091: connect: connection refused I0703 18:02:32.559391 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 5.940996ms to process 0 records I0703 18:02:32.559401 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 19.6834ms to process 0 records E0703 18:02:32.559409 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0703 18:02:32.559416 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 19.890321ms to process 0 records I0703 18:02:32.559425 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 20.05714ms to process 0 records I0703 18:02:32.559468 1 tasks_processing.go:71] worker 31 working on install_plans task. I0703 18:02:32.560034 1 tasks_processing.go:74] worker 6 stopped. I0703 18:02:32.560098 1 tasks_processing.go:71] worker 22 working on ceph_cluster task. I0703 18:02:32.560148 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=610003d420c8ecc38eb63a25396cca5565774601c95df9496ecb61f23d275463 E0703 18:02:32.560185 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.134.146:9091: connect: connection refused I0703 18:02:32.560307 1 tasks_processing.go:74] worker 24 stopped. I0703 18:02:32.560435 1 tasks_processing.go:74] worker 30 stopped. I0703 18:02:32.560556 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=7135825ca93ee2a0d1d8744129ad6f6010f07213e67a08c5402d54c1c81e565d I0703 18:02:32.560573 1 gather.go:177] gatherer "clusterconfig" function "crds" took 42.472081ms to process 2 records I0703 18:02:32.560581 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 28.066432ms to process 0 records I0703 18:02:32.560598 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0703 18:02:32.560606 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 43.308312ms to process 1 records E0703 18:02:32.560614 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.134.146:9091: connect: connection refused I0703 18:02:32.560641 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 5.356056ms to process 0 records I0703 18:02:32.560661 1 tasks_processing.go:74] worker 28 stopped. I0703 18:02:32.560668 1 tasks_processing.go:74] worker 18 stopped. I0703 18:02:32.560673 1 tasks_processing.go:74] worker 23 stopped. I0703 18:02:32.566520 1 tasks_processing.go:74] worker 26 stopped. I0703 18:02:32.566541 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 13.964745ms to process 0 records I0703 18:02:32.569113 1 tasks_processing.go:74] worker 13 stopped. I0703 18:02:32.569226 1 recorder.go:75] Recording config/image with fingerprint=bfb65952be1a47b50f2c63cbc506804491a5e3d11418ef40096dbdc73cb8208c I0703 18:02:32.569243 1 gather.go:177] gatherer "clusterconfig" function "image" took 18.878276ms to process 1 records I0703 18:02:32.569507 1 tasks_processing.go:74] worker 1 stopped. I0703 18:02:32.569748 1 recorder.go:75] Recording config/ingress with fingerprint=ced6260c352bc4381f4f70797c70ba6f21b6f87eb6df50c01f190882dc554043 I0703 18:02:32.569769 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 31.601496ms to process 1 records I0703 18:02:32.569779 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 13.841696ms to process 0 records I0703 18:02:32.569791 1 tasks_processing.go:74] worker 16 stopped. I0703 18:02:32.569907 1 tasks_processing.go:74] worker 15 stopped. I0703 18:02:32.569988 1 recorder.go:75] Recording config/proxy with fingerprint=a4ce7d77708254315146012b8e68cc6f76513ba9949597e7696bf67c3475bd20 I0703 18:02:32.570011 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 18.446217ms to process 1 records I0703 18:02:32.570070 1 tasks_processing.go:74] worker 19 stopped. I0703 18:02:32.570607 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=ba5c9db23e0b5b8623d5681fc291cf1defd3e978d73e047325a2d3de4cf5141d I0703 18:02:32.570622 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 20.408547ms to process 1 records I0703 18:02:32.573941 1 tasks_processing.go:74] worker 20 stopped. I0703 18:02:32.574193 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=1c89629652662e58bdbf2d0096b5db89c9bebc2caa4fbe805de039ae82601e17 I0703 18:02:32.574350 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=918d1771a33abcadc54d870059907c613d58b2efedd331652da899c618a03ae5 I0703 18:02:32.574370 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 49.819418ms to process 2 records I0703 18:02:32.574463 1 tasks_processing.go:74] worker 4 stopped. I0703 18:02:32.574535 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=4e9ea5cbbbc7e57652bc21d5222220a909d06f6b90f43273d375c2a71da53566 I0703 18:02:32.574618 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=4066c4ad02b456f6a13710d7de71846c46a732455cb6991f1673e5e6fbe839f9 I0703 18:02:32.574760 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=7c3d0ad45fb42502aab6d7fb339a493f7d8a34bdf44313824c41f287103fd871 I0703 18:02:32.574788 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=ae545c13c863f380806096d1ce9b0e0574c27376a44600e00d94d3e88df5c31f I0703 18:02:32.574811 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=bc6f7cdada2fe0a859319c3105f4c2ff902d20c235a916a1bac6346e44673d46 I0703 18:02:32.574835 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=31762f5675f7af2e7204bfa5389173db81c9f636cf9bcb7fd9ec1e238639549b I0703 18:02:32.574858 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=93560527d1a9bcd78a45954c21de8c910f7c6b5951d9c1e69c115e373a4428e4 I0703 18:02:32.574881 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=5c5cca6d5e84eb5d3ee76cf4471385ef71703a2c02122cc17b3056282bd76066 I0703 18:02:32.574904 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=16d88f96138f331edd37cbfd4008fbcd0c143a2a091ec52ae20a74c168542d39 I0703 18:02:32.574937 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=cf4f96ec125c648c82f1414011aa78b922c571227e5d3024f84df08b17d8ed69 I0703 18:02:32.574961 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=d0099564a4f3caddd202e89dacfee6b48ff93fd1c202cbf43a36a00812598327 I0703 18:02:32.574984 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=92e108fb47c96bd79cff69236de56747dce7ec4fce8f5ef6e9de88da1189267e I0703 18:02:32.575008 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=cc27cfc95b085a789d54daeb5e5916724681c7b5733fac75c2430f3f2477cb79 I0703 18:02:32.575017 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 24.664759ms to process 13 records I0703 18:02:32.582647 1 tasks_processing.go:74] worker 29 stopped. I0703 18:02:32.582667 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 26.124538ms to process 0 records I0703 18:02:32.584491 1 tasks_processing.go:74] worker 7 stopped. I0703 18:02:32.584682 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=d15f54a46765ff3cd2d5931390e0f451276e9ecc8b1470eb54e11015e441842a I0703 18:02:32.584748 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=6e7e477c0465e0ef14a593d29c9a8aeb4fc2afb5e11b15126ea879e44d3e8301 I0703 18:02:32.584792 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=c769f2c710d320848feae443ed1e415f8fc766c1d797e9075ed57ff42ebb7b73 I0703 18:02:32.584805 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 27.985019ms to process 3 records I0703 18:02:32.585933 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0703 18:02:32.587315 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0703 18:02:32.587434 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0703 18:02:32.587442 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0703 18:02:32.589525 1 tasks_processing.go:74] worker 12 stopped. I0703 18:02:32.590001 1 recorder.go:75] Recording config/version with fingerprint=6d2e165c5c0b922a0afb615dd2b29a7ef11262597b15c8f6d3330a72d143dfb9 I0703 18:02:32.590021 1 recorder.go:75] Recording config/id with fingerprint=4abd31ecd774fd24aafcca755f4d67c50c90da9cb7e518c291c0df8b73128370 I0703 18:02:32.590030 1 gather.go:177] gatherer "clusterconfig" function "version" took 52.766135ms to process 2 records I0703 18:02:32.590039 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 29.388103ms to process 0 records I0703 18:02:32.590048 1 tasks_processing.go:74] worker 22 stopped. I0703 18:02:32.590073 1 tasks_processing.go:74] worker 0 stopped. I0703 18:02:32.590241 1 recorder.go:75] Recording config/network with fingerprint=f95eaa56bda750e354f065103648d1a31b7faf2c3449603d51c9d854e424a9fb I0703 18:02:32.590314 1 gather.go:177] gatherer "clusterconfig" function "networks" took 33.475562ms to process 1 records I0703 18:02:32.592534 1 base_controller.go:82] Caches are synced for ConfigController I0703 18:02:32.592563 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0703 18:02:32.595997 1 tasks_processing.go:74] worker 2 stopped. I0703 18:02:32.596037 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0703 18:02:32.596104 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 46.049592ms to process 1 records I0703 18:02:32.596786 1 tasks_processing.go:74] worker 11 stopped. I0703 18:02:32.596807 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 41.537181ms to process 0 records I0703 18:02:32.599434 1 tasks_processing.go:74] worker 8 stopped. I0703 18:02:32.600752 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-575cd97545-x69tj with fingerprint=645ba453eebaed2bb9932be8feb4ba96882139b127116af6721bcfdd3c2bbf57 I0703 18:02:32.601082 1 recorder.go:75] Recording config/pod/openshift-ovn-kubernetes/ovnkube-node-86jrk with fingerprint=4cb2776d818446580c13e24796da10521acb90639495bf6f0dec0bd72af6a22c I0703 18:02:32.601154 1 recorder.go:75] Recording config/running_containers with fingerprint=f5549974685065db90b19adedfdd8223409e871b1f1a6dce1570d23416be539e I0703 18:02:32.601163 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 81.285321ms to process 3 records E0703 18:02:32.601176 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0703 18:02:32.601185 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2raqdrbpug8ib1bhcvajrpslq5r1odtg-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2raqdrbpug8ib1bhcvajrpslq5r1odtg-primary-cert-bundle-secret" not found I0703 18:02:32.601223 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=5c9834478458f8eb7472f7f42c4e106e9f6e97bbd916eb0562bff9106fb39c3b I0703 18:02:32.601232 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 83.372071ms to process 1 records I0703 18:02:32.601240 1 tasks_processing.go:74] worker 21 stopped. I0703 18:02:32.602748 1 configmapobserver.go:84] configmaps "insights-config" not found E0703 18:02:32.609953 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0703 18:02:32.609970 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0703 18:02:32.609976 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0703 18:02:32.609977 1 tasks_processing.go:74] worker 5 stopped. I0703 18:02:32.609990 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=26b6661162b099a0f5a279859b4f46c867929a79d9a4a41fde4be4e6fe138018 I0703 18:02:32.610060 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=f73fcd48d6f9ebbc83253a1b5f23ef59d0683465ec46e474389103c717467dfb I0703 18:02:32.610073 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0703 18:02:32.610079 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=c93090eb0d2a4736885abeb79c91680cfd01fda46464f83456b085d4dc8239f0 I0703 18:02:32.610091 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0703 18:02:32.610143 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0703 18:02:32.610153 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0703 18:02:32.610160 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 93.247009ms to process 7 records I0703 18:02:32.618259 1 tasks_processing.go:74] worker 17 stopped. I0703 18:02:32.618325 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 61.127303ms to process 0 records I0703 18:02:32.625188 1 base_controller.go:82] Caches are synced for LoggingSyncer I0703 18:02:32.625202 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0703 18:02:32.629357 1 gather_cluster_operator_pods_and_events.go:121] Found 6 pods with 6 containers I0703 18:02:32.629378 1 gather_cluster_operator_pods_and_events.go:235] Maximum buffer size: 4194304 bytes I0703 18:02:32.629429 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-857d7d997-5n9l6 pod in namespace openshift-ingress (previous: false). I0703 18:02:32.633870 1 cluster_transfer.go:102] no available accepted cluster transfer I0703 18:02:32.633883 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0703 18:02:32.646130 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-85f6d54bb9-kj5d9 pod in namespace openshift-ingress (previous: false). I0703 18:02:32.662334 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-85f6d54bb9-npq2s pod in namespace openshift-ingress (previous: false). I0703 18:02:32.664912 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0703 18:02:32.675498 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0703 18:02:32.683690 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-5l62l pod in namespace openshift-ingress-canary (previous: false). I0703 18:02:32.693782 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-74nnf pod in namespace openshift-ingress-canary (previous: false). I0703 18:02:32.761829 1 sca.go:266] etc-pki-entitlement secret successfully created I0703 18:02:32.761847 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated I0703 18:02:32.772358 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-k9sd2 pod in namespace openshift-ingress-canary (previous: false). I0703 18:02:32.971983 1 tasks_processing.go:74] worker 9 stopped. I0703 18:02:32.972083 1 recorder.go:75] Recording events/openshift-ingress-operator with fingerprint=c28626381c869233fac417e122acbf2f195d9a3945719f13edefd66d2964953e I0703 18:02:32.972189 1 recorder.go:75] Recording events/openshift-ingress with fingerprint=d00276605d7cb50997f9062812902e18d48f99b66fdf86321338e3e59daa3cb6 I0703 18:02:32.972236 1 recorder.go:75] Recording events/openshift-ingress-canary with fingerprint=4186b44fb81c997d1ba6f2819eef713c46589db719b14cc68186dc883bba5866 I0703 18:02:32.972275 1 recorder.go:75] Recording config/pod/openshift-ingress/logs/router-default-857d7d997-5n9l6/router_current.log with fingerprint=967ad5fcfcffbe8beaa76b92b328bbd363fec422edd928ec3bb22a0deda6e230 I0703 18:02:32.972339 1 recorder.go:75] Recording config/pod/openshift-ingress/logs/router-default-85f6d54bb9-kj5d9/router_current.log with fingerprint=a7e77e1f02823ae3abbe1251317037b9fb315b51a8542906d1db3b6516b731f5 I0703 18:02:32.972376 1 recorder.go:75] Recording config/pod/openshift-ingress/logs/router-default-85f6d54bb9-npq2s/router_current.log with fingerprint=cfe114e50c96145a3137754d1bc854991b1495e58c195bd79eb6f7f1bc81eb47 I0703 18:02:32.972383 1 recorder.go:75] Recording config/pod/openshift-ingress-canary/logs/ingress-canary-5l62l/serve-healthcheck-canary_current.log with fingerprint=490c80ecf92f067d670ee3491e29d53ec62fc333573285bb6ff71f484fda96bd I0703 18:02:32.972387 1 recorder.go:75] Recording config/pod/openshift-ingress-canary/logs/ingress-canary-74nnf/serve-healthcheck-canary_current.log with fingerprint=c6a3afd79a8f45adf90aa997eb43df8a26959271528f5e6e935aa4dad800cddc I0703 18:02:32.972391 1 recorder.go:75] Recording config/pod/openshift-ingress-canary/logs/ingress-canary-k9sd2/serve-healthcheck-canary_current.log with fingerprint=adbc3c9a4d9738e8c9e44e40a3a633fead7fa99e6ae526f5d1a3188b2969dce5 I0703 18:02:32.972398 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 453.685997ms to process 9 records W0703 18:02:33.540230 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0703 18:02:33.550727 1 tasks_processing.go:74] worker 3 stopped. I0703 18:02:33.550754 1 recorder.go:75] Recording config/silenced_alerts.json with fingerprint=37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570 I0703 18:02:33.550776 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 1.033792221s to process 1 records I0703 18:02:33.550850 1 tasks_processing.go:74] worker 25 stopped. I0703 18:02:33.550899 1 recorder.go:75] Recording config/alerts with fingerprint=2a4d5ad90b87adbb3fd2cefcc2b7d0b4d30738bc0813ecd4862ec02ae898bc11 I0703 18:02:33.550918 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 1.034602569s to process 1 records W0703 18:02:34.540332 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0703 18:02:35.018852 1 tasks_processing.go:74] worker 14 stopped. I0703 18:02:35.018975 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=8500a289c93510bbba537ff693be2f819a6f879f8b5109ef5f19c17cb91adc3f I0703 18:02:35.019024 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0703 18:02:35.019050 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=ab5eeee709e3852b7e592c23bd08c23a60e56095ab4d211152064e867279afa9 I0703 18:02:35.019060 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0703 18:02:35.019081 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=77a85de6659cc31af6ebf04008bcf7102c29c9057c62814be81450fbdbe1c080 I0703 18:02:35.019096 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0703 18:02:35.019117 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=8f58e008dc00fb85822fe1cd68984544722631849169869a2be85082d3a04dae I0703 18:02:35.019146 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=dc28da87f36ef5a071d85112afbf2b05a1ba00aa2cff44eec7f0f0b6bc62ef53 I0703 18:02:35.019167 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=0e2edada5bcc149c4fcadd4760e335064bb61c653a1450a2b1c349d636ef7a9b I0703 18:02:35.019198 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=a15de53e0ecf5ef8089a230774c43ace7b9c1e744af0e686fdb679779d7e908c I0703 18:02:35.019457 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=1123a1240ecf78e05904216ff51fb73de7c944c0f1b375565328e21208dac76f I0703 18:02:35.019477 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=9583ea5d1d71efd60675761547c5ca1d14279ab6fd25ce65723a81da2af7493c I0703 18:02:35.019485 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0703 18:02:35.019502 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=6f254d5005fbaf8537312170dbe3d64e7174340dcda2318285e8a4b6362d294c I0703 18:02:35.019510 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0703 18:02:35.019526 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=89a6648849aed9eccbb9a4c13b2de978619c157d330fa9b22b9d8df0cbf155df I0703 18:02:35.019535 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0703 18:02:35.019554 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=114b7ffb104386110d08dda719b3ee184ae6a46321708deab1c897dae212536b I0703 18:02:35.019563 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0703 18:02:35.019582 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=747153a1450eb6084c981e0c132c76d048d1fb52022ba88a081e971aa38ea1ec I0703 18:02:35.019685 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=1504a15a559c411508cea09c343cce90809817274bef751662a6eaa1ade52fe4 I0703 18:02:35.019694 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0703 18:02:35.019715 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0703 18:02:35.019736 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0703 18:02:35.019757 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=f44d2cbfb9ba39f95d0dd834fd4450e3a78e4255e7d235f04557852835c1d956 I0703 18:02:35.019781 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=9b905dbf5387cfb2bbc8e3280f6885831d0a4a14440782b5ea34e8ca42203aaa I0703 18:02:35.019791 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0703 18:02:35.019806 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=58437ff1323416f123bc70a5cfa942b5ae62c195f8f7c1fe92020986c6f47045 I0703 18:02:35.019815 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0703 18:02:35.019833 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=d45472c6dab45c4b0ab542d497390e1949e1fff1d0adeb55ef04f3147937614a I0703 18:02:35.019842 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0703 18:02:35.019854 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=ca2fcf6f22bb569ef60fcfd03b5191377141c73ba9ebd655a7a6d931badd72e7 I0703 18:02:35.019871 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=96eb86add76836dfaacaccf189fa27b047afcf8c109cf645409c2f47ca78c300 I0703 18:02:35.019886 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=5e1ee6fd888b5ac198b884fde172dee98fea1337191e38bd38dd05988660d64a I0703 18:02:35.019907 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=2d971b6a8ee0a198ae46dba6d7d9d24ee923dbae88bb4e12488813b128c32e38 I0703 18:02:35.019917 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0703 18:02:35.019943 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=5fcf81123fd31a58d4cdef1367d0c109ad43e5b47c1bd1daf2a853abeb0e1dbd I0703 18:02:35.019959 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0703 18:02:35.019966 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0703 18:02:35.019996 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.500805225s to process 39 records W0703 18:02:35.539606 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0703 18:02:36.540210 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0703 18:02:37.538486 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0703 18:02:37.538511 1 tasks_processing.go:74] worker 27 stopped. E0703 18:02:37.538522 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0703 18:02:37.538533 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0703 18:02:37.538545 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.0221501s to process 1 records I0703 18:02:45.594822 1 tasks_processing.go:74] worker 31 stopped. I0703 18:02:45.594860 1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b I0703 18:02:45.594873 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 13.035307268s to process 1 records I0703 18:02:46.345112 1 tasks_processing.go:74] worker 10 stopped. I0703 18:02:46.345434 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=b5628417db3123c06333720ebefb8c69f4980e1afaf323f1ff9cb97dfe717013 I0703 18:02:46.345449 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.805181186s to process 1 records E0703 18:02:46.345518 1 periodic.go:247] "Unhandled Error" err="clusterconfig failed after 13.83s with: function \"pod_network_connectivity_checks\" failed with an error, function \"support_secret\" failed with an error, function \"machines\" failed with an error, function \"tsdb_status\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"metrics\" failed with an error, function \"ingress_certificates\" failed with an error, function \"config_maps\" failed with an error, function \"dvo_metrics\" failed with an error" I0703 18:02:46.346629 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "pod_network_connectivity_checks" failed with an error, function "support_secret" failed with an error, function "machines" failed with an error, function "tsdb_status" failed with an error, function "machine_healthchecks" failed with an error, function "metrics" failed with an error, function "ingress_certificates" failed with an error, function "config_maps" failed with an error, function "dvo_metrics" failed with an error I0703 18:02:46.346646 1 periodic.go:209] Running workloads gatherer I0703 18:02:46.346666 1 tasks_processing.go:45] number of workers: 2 I0703 18:02:46.346682 1 tasks_processing.go:69] worker 1 listening for tasks. I0703 18:02:46.346689 1 tasks_processing.go:71] worker 1 working on workload_info task. I0703 18:02:46.346691 1 tasks_processing.go:69] worker 0 listening for tasks. I0703 18:02:46.346725 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0703 18:02:46.352208 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-109.ec2.internal... I0703 18:02:46.352217 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-113.ec2.internal... I0703 18:02:46.352208 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-165.ec2.internal... I0703 18:02:46.366929 1 tasks_processing.go:74] worker 0 stopped. I0703 18:02:46.366945 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 20.189676ms to process 0 records I0703 18:02:51.362753 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 5s I0703 18:02:51.388487 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0703 18:02:51.396013 1 gather_workloads_info.go:387] No image sha256:765f0d23b637f685f98a31bd47c131b03cf72a40761a3f9a9d6320faa3c33733 (9ms) I0703 18:02:51.403766 1 gather_workloads_info.go:387] No image sha256:04c87c054a3f366a7dfbe0a93ebb0c80a098ee16842c5794b67c1202eec61996 (8ms) I0703 18:02:51.412358 1 gather_workloads_info.go:387] No image sha256:2904a78e2eb73fd6a9bb94c105c2a056831fb4113fbb7b0607c50adc9d879c9b (9ms) I0703 18:02:51.420364 1 gather_workloads_info.go:387] No image sha256:513b1166e376feca472fdd5174f6abf0abf02925d6a21b7b4d89a2ff2fc67bed (8ms) I0703 18:02:51.429023 1 gather_workloads_info.go:387] No image sha256:653c666f842c13e0baae2e89a9b1efe0e2ef56f621ffb5b32005115d2a26ab8c (9ms) I0703 18:02:51.438410 1 gather_workloads_info.go:387] No image sha256:50197f22710766515f67944a779e00dd9ae3d17b18732d7324a970353b11f292 (9ms) I0703 18:02:51.446372 1 gather_workloads_info.go:387] No image sha256:5a95c19d82767e0235b4edb4a0536482c816904897aae1dc3eb255cb52b87a9f (8ms) I0703 18:02:51.454577 1 gather_workloads_info.go:387] No image sha256:a498046d64605bcccee2440aa4f04a4602baaae263cf01d977ec5208e876b1fd (8ms) I0703 18:02:51.462656 1 gather_workloads_info.go:387] No image sha256:a258c226562adb14e3a163a1940938526ee6a0928982a7667d85d9a7334ce639 (8ms) I0703 18:02:51.470420 1 gather_workloads_info.go:387] No image sha256:695cf2f0cc07683c2a3ce1eaf3e56fe18abc6e2bac716f7d9843f5d173b9df52 (8ms) I0703 18:02:51.495945 1 gather_workloads_info.go:387] No image sha256:101993f6e65d6fda8b90c583b718b720809182961f69bc777295f47e6df4d7ce (26ms) I0703 18:02:51.596010 1 gather_workloads_info.go:387] No image sha256:74713675d580a47a56b628469c4c05c41e114e0b935960d718ff0cf7e72f613d (100ms) I0703 18:02:51.697089 1 gather_workloads_info.go:387] No image sha256:521712486e2c6e3c020dad6a1cb340db8e55665b69f7c208fab9cd9e965fd588 (101ms) I0703 18:02:51.796684 1 gather_workloads_info.go:387] No image sha256:36b9e89c3cfcf1ab9ae500486e38afb6862cba48cb0b4d84a09508ab8f3d299f (100ms) I0703 18:02:51.897362 1 gather_workloads_info.go:387] No image sha256:0a99240166165eb5718e7516a43282fe32df9c7c5e809b31b58abe44e42ff94d (101ms) I0703 18:02:51.996242 1 gather_workloads_info.go:387] No image sha256:3d6ea9e1633f38070718a1357fcb80e6cb3932aafcc4d65e659795b186c5da15 (99ms) I0703 18:02:52.096121 1 gather_workloads_info.go:387] No image sha256:7adc1eab05d6724c76ba751f6df816b08d6e70b78dee9eb94fa6fd9690542c98 (100ms) I0703 18:02:52.198363 1 gather_workloads_info.go:387] No image sha256:943018739e3db1763c3184b460dbc409e058abbac76d57b9927faad317be85e4 (102ms) I0703 18:02:52.296292 1 gather_workloads_info.go:387] No image sha256:c940ea87e7d133d75ba0002ef00c0806825eed3db8094cdb260d1bac18127733 (98ms) I0703 18:02:52.396513 1 gather_workloads_info.go:387] No image sha256:a0105d1eb62cf6ac9e5e2ef28d3e89bf6dc514bc594fc7090fe5a5ee18a09c87 (100ms) I0703 18:02:52.496288 1 gather_workloads_info.go:387] No image sha256:76cc47f3775560477ec548c7b6fddbd99112ad6604f790697eef546576051fb7 (100ms) I0703 18:02:52.596176 1 gather_workloads_info.go:387] No image sha256:1a2532940843248c57d52141185dd71fbc393ab28b65d48f682038632c1dbbad (100ms) I0703 18:02:52.695964 1 gather_workloads_info.go:387] No image sha256:56a85660a445eced5c79a595a0eccf590087c5672d50f49d4c25ad52f9a44f04 (100ms) I0703 18:02:52.796718 1 gather_workloads_info.go:387] No image sha256:47154813651033d59751fb655a384dbffb64dd26f10bd7f3be0c3128d0486356 (101ms) I0703 18:02:52.896857 1 gather_workloads_info.go:387] No image sha256:90bdf9e678fe18e35401e34e1cb0912d822e9cb32b9e41cc545870540e9a87f2 (100ms) I0703 18:02:52.995478 1 gather_workloads_info.go:387] No image sha256:7bd3e88d6d3f878a89203df590b684c5a77bf08df4717451d3197ab74683a80e (99ms) I0703 18:02:53.096024 1 gather_workloads_info.go:387] No image sha256:95418d0ba6fc0aa96647d97cf6530967f7d7ae97a82ce7f7e41c8e44310b0b72 (101ms) I0703 18:02:53.198055 1 gather_workloads_info.go:387] No image sha256:1e65f9f5490fbc930d2dce96f03ddcb28990b839d7dc011ef5de5bec6c5095bf (102ms) I0703 18:02:53.295855 1 gather_workloads_info.go:387] No image sha256:0092c6ff139f8698c91349d507eb3182bc5d6c4ae040f672e347ff504accdea2 (98ms) I0703 18:02:53.397900 1 gather_workloads_info.go:387] No image sha256:1477f3287cc75352305274f248f781c1b7c2a8696f4f7bc450c45a4a5f5b10ea (102ms) I0703 18:02:53.495600 1 gather_workloads_info.go:387] No image sha256:3612c5a433effd17b7aff7ff6edf4fb27b61153908e3a0294158e4886a1d600d (98ms) I0703 18:02:53.596767 1 gather_workloads_info.go:387] No image sha256:03cf4cd7ef1518610c6c7b3ad27d1622d82e98e3dc6e3f8e5d0fceb5c8d3786e (101ms) I0703 18:02:53.696916 1 gather_workloads_info.go:387] No image sha256:a56211d075aa43cbb491f669a5b2e46ee023dc95b7d51dbac28f463948c5ad61 (100ms) I0703 18:02:53.796742 1 gather_workloads_info.go:387] No image sha256:91828234f107c068c8a4966d08370ae7b73e637651dbc6d92c18c4553402c22c (100ms) I0703 18:02:53.897026 1 gather_workloads_info.go:387] No image sha256:2e564f336c77116053f34d4201d364d8da04e789cfffa0ea422574c95f2d6404 (100ms) I0703 18:02:53.996279 1 gather_workloads_info.go:387] No image sha256:2e57e192c3c1240fd935dcd55c8fde5e70e78bf81d6176c96edf21fafe59f8ba (99ms) I0703 18:02:54.097200 1 gather_workloads_info.go:387] No image sha256:b3909bf664c77097f75b3768830863d642eed3815dab2bfb4415c771ca2d5007 (101ms) I0703 18:02:54.196344 1 gather_workloads_info.go:387] No image sha256:4556896f77307821531ef91b7b7faccb82b824ea695693b2989f597f0deca038 (99ms) I0703 18:02:54.296268 1 gather_workloads_info.go:387] No image sha256:32d4561d4bf1b3c4e3a7878f918f9c1422bab85bdf8711fbdf905d59692fee81 (100ms) I0703 18:02:54.396594 1 gather_workloads_info.go:387] No image sha256:5f0b67cfbbc381243fb91ccc17345b56d05f4d717c667e8c644e5bf05633ba71 (100ms) I0703 18:02:54.497394 1 gather_workloads_info.go:387] No image sha256:2598489729a4b258e4ecda4a06f6875133f2a10ced5c5241f8a57a8a05418e36 (101ms) I0703 18:02:54.597332 1 gather_workloads_info.go:387] No image sha256:6e5df2aab9ce759e8f957341a11c087c233307e3c8f8760d21270efc19ccd3ce (100ms) I0703 18:02:54.699334 1 gather_workloads_info.go:387] No image sha256:0af73ad7a47c9ff53269ae2bc5163bf9e389085220c34e4bd42c92c4e7a85f6d (102ms) I0703 18:02:54.796748 1 gather_workloads_info.go:387] No image sha256:7b31223098f08328f5ddea8e5b871dbbd5f5a61ec550e8956f66793c0c6031a9 (97ms) I0703 18:02:54.896663 1 gather_workloads_info.go:387] No image sha256:289816958633a763a72dbc44e1dad40466223164e7e253039514f0d974ea5d21 (100ms) I0703 18:02:54.997017 1 gather_workloads_info.go:387] No image sha256:b99613494e9d6a4a8207323872d63cc02b830fc80a8bf972e62c64381aa4cd17 (100ms) I0703 18:02:55.099368 1 gather_workloads_info.go:387] No image sha256:25bda9d34e23ba6bc9df5cf7104bdb237269740a3a76ee025f88b34e68b7e2b5 (102ms) I0703 18:02:55.099417 1 tasks_processing.go:74] worker 1 stopped. I0703 18:02:55.099778 1 recorder.go:75] Recording config/workload_info with fingerprint=66390addb509852c9b2c228bdc7cce23abf5d30ebe73c813d4c68d132b717189 I0703 18:02:55.099794 1 gather.go:177] gatherer "workloads" function "workload_info" took 8.752704668s to process 1 records I0703 18:02:55.099808 1 periodic.go:256] Periodic gather workloads completed in 8.753s I0703 18:02:55.099816 1 controllerstatus.go:80] name=periodic-workloads healthy=true reason= message= I0703 18:02:55.099821 1 periodic.go:209] Running conditional gatherer I0703 18:02:55.105680 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules I0703 18:02:55.195337 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer E0703 18:02:55.197779 1 conditional_gatherer.go:324] unable to update alerts cache: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?match%5B%5D=ALERTS%7Balertstate%3D%22firing%22%7D&query=ALERTS": dial tcp 172.30.134.146:9091: connect: connection refused I0703 18:02:55.197855 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0703 18:02:55.204080 1 conditional_gatherer.go:392] cluster version is '4.20.8' E0703 18:02:55.204095 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0703 18:02:55.204100 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0703 18:02:55.204103 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0703 18:02:55.204106 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0703 18:02:55.204109 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0703 18:02:55.204112 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0703 18:02:55.204114 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0703 18:02:55.204117 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0703 18:02:55.204120 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0703 18:02:55.204123 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0703 18:02:55.204138 1 tasks_processing.go:45] number of workers: 3 I0703 18:02:55.204144 1 tasks_processing.go:69] worker 2 listening for tasks. I0703 18:02:55.204148 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0703 18:02:55.204156 1 tasks_processing.go:69] worker 0 listening for tasks. I0703 18:02:55.204172 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0703 18:02:55.204172 1 tasks_processing.go:69] worker 1 listening for tasks. I0703 18:02:55.204180 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0703 18:02:55.204182 1 tasks_processing.go:74] worker 1 stopped. I0703 18:02:55.204226 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=9fafdcbf373f1f367f1d69551ea59114592cce1681189be0f3dde774e0530661 I0703 18:02:55.204240 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 868ns to process 1 records I0703 18:02:55.204274 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0703 18:02:55.204281 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.42µs to process 1 records I0703 18:02:55.204287 1 tasks_processing.go:74] worker 0 stopped. I0703 18:02:55.204454 1 tasks_processing.go:74] worker 2 stopped. I0703 18:02:55.204470 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 268.001µs to process 0 records I0703 18:02:55.204491 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0703 18:02:55.204517 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0703 18:02:55.229860 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=c5eed84457f39dc42405539a370d9a5ec9dd7d2f2b58bad5e0c1a4bea5208146 I0703 18:02:55.229983 1 diskrecorder.go:70] Writing 120 records to /var/lib/insights-operator/insights-2026-07-03-180255.tar.gz I0703 18:02:55.237895 1 diskrecorder.go:51] Wrote 120 records to disk in 7ms I0703 18:02:55.237926 1 periodic.go:278] Gathering cluster info every 2h0m0s I0703 18:02:55.237942 1 periodic.go:279] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0703 18:04:02.515734 1 diskrecorder.go:223] Found files to send: insights-2026-07-03-180255.tar.gz I0703 18:04:02.515763 1 insightsuploader.go:150] Checking archives to upload periodically every 15m47.50171473s I0703 18:04:02.515772 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0703 18:04:02.530598 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0703 18:04:02.630884 1 requests.go:87] Successfully reported id=2026-07-03T18:04:02Z x-rh-insights-request-id=9c06c70c5ad341a3a93a9240807fb43e, wrote=58311 I0703 18:04:02.631004 1 insightsuploader.go:187] Uploaded report successfully in 115.221331ms I0703 18:04:02.631026 1 controller.go:128] Initializing last reported time to 2026-07-03T18:04:02Z I0703 18:04:02.631095 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0703 18:04:02.631103 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0703 18:04:02.631110 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0703 18:04:02.636044 1 controller.go:512] The operator is healthy I0703 18:04:32.530907 1 controller.go:512] The operator is healthy I0703 18:05:05.888607 1 insightsreport.go:137] Pulling report from smart-proxy I0703 18:05:05.888636 1 insightsreport.go:149] Retrieving report I0703 18:05:05.895168 1 requests.go:110] Retrieving report for cluster: cac93a23-e1d6-4328-8190-cd445ceb023a I0703 18:05:05.895182 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/cac93a23-e1d6-4328-8190-cd445ceb023a/reports I0703 18:05:05.898009 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/cac93a23-e1d6-4328-8190-cd445ceb023a/reports I0703 18:05:06.105595 1 insightsreport.go:184] Report retrieved I0703 18:05:06.143349 1 insightsreport.go:239] Report retrieved correctly I0703 18:06:32.530485 1 controller.go:512] The operator is healthy I0703 18:07:32.508375 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0703 18:07:32.516837 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0703 18:07:32.516857 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0703 18:07:32.520501 1 secretconfigobserver.go:119] support secret does not exist I0703 18:08:32.532964 1 controller.go:512] The operator is healthy I0703 18:10:32.530397 1 controller.go:512] The operator is healthy I0703 18:12:32.521167 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0703 18:12:32.592278 1 configmapobserver.go:84] configmaps "insights-config" not found I0703 18:12:32.592588 1 controller.go:512] The operator is healthy I0703 18:12:32.608455 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0703 18:12:32.608473 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0703 18:12:32.646277 1 secretconfigobserver.go:119] support secret does not exist I0703 18:14:32.532492 1 controller.go:512] The operator is healthy I0703 18:16:32.535017 1 controller.go:512] The operator is healthy I0703 18:17:32.646977 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0703 18:17:32.654407 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0703 18:17:32.654427 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0703 18:17:32.658112 1 secretconfigobserver.go:119] support secret does not exist I0703 18:18:32.534264 1 controller.go:512] The operator is healthy I0703 18:19:55.134061 1 insightsuploader.go:146] Nothing to report since 2026-07-03T18:04:02Z I0703 18:20:32.534715 1 controller.go:512] The operator is healthy I0703 18:22:32.514999 1 configmapobserver.go:84] configmaps "insights-config" not found I0703 18:22:32.530290 1 controller.go:512] The operator is healthy I0703 18:22:32.658492 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0703 18:22:32.662299 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0703 18:22:32.662320 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0703 18:22:32.665519 1 secretconfigobserver.go:119] support secret does not exist I0703 18:24:32.535346 1 controller.go:512] The operator is healthy I0703 18:26:32.530674 1 controller.go:512] The operator is healthy I0703 18:27:32.665819 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0703 18:27:32.673732 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0703 18:27:32.673756 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0703 18:27:32.677194 1 secretconfigobserver.go:119] support secret does not exist I0703 18:28:32.534280 1 controller.go:512] The operator is healthy I0703 18:30:32.533292 1 controller.go:512] The operator is healthy I0703 18:32:32.508783 1 configmapobserver.go:84] configmaps "insights-config" not found I0703 18:32:32.512251 1 configmapobserver.go:84] configmaps "insights-config" not found I0703 18:32:32.530835 1 controller.go:512] The operator is healthy I0703 18:32:32.677912 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0703 18:32:32.682033 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0703 18:32:32.682074 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0703 18:32:32.685306 1 secretconfigobserver.go:119] support secret does not exist I0703 18:34:32.531387 1 controller.go:512] The operator is healthy I0703 18:36:29.637049 1 insightsuploader.go:146] Nothing to report since 2026-07-03T18:04:02Z I0703 18:36:32.535625 1 controller.go:512] The operator is healthy I0703 18:37:32.686223 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0703 18:37:32.694507 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0703 18:37:32.694527 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0703 18:37:32.698457 1 secretconfigobserver.go:119] support secret does not exist I0703 18:38:32.532940 1 controller.go:512] The operator is healthy I0703 18:40:32.534878 1 controller.go:512] The operator is healthy I0703 18:42:32.510949 1 configmapobserver.go:84] configmaps "insights-config" not found I0703 18:42:32.531008 1 controller.go:512] The operator is healthy I0703 18:42:32.699184 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0703 18:42:32.703962 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0703 18:42:32.703981 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0703 18:42:32.707882 1 secretconfigobserver.go:119] support secret does not exist I0703 18:44:32.536042 1 controller.go:512] The operator is healthy I0703 18:46:32.533970 1 controller.go:512] The operator is healthy I0703 18:47:32.707999 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0703 18:47:32.715214 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0703 18:47:32.715234 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0703 18:47:32.719174 1 secretconfigobserver.go:119] support secret does not exist I0703 18:48:32.537328 1 controller.go:512] The operator is healthy