W0704 13:59:21.279277 1 cmd.go:257] Using insecure, self-signed certificates I0704 13:59:21.754254 1 start.go:138] Unable to read service ca bundle: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0704 13:59:21.754537 1 observer_polling.go:159] Starting file observer I0704 13:59:22.374703 1 operator.go:60] Starting insights-operator v0.0.0-master+$Format:%H$ I0704 13:59:22.374911 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0704 13:59:22.375463 1 secure_serving.go:57] Forcing use of http/1.1 only W0704 13:59:22.375485 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0704 13:59:22.375489 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0704 13:59:22.375512 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0704 13:59:22.375516 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0704 13:59:22.375519 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0704 13:59:22.375521 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0704 13:59:22.375561 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0704 13:59:22.383233 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"60a94830-78a8-4dde-a2b9-e2db1b17c651", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}} I0704 13:59:22.383228 1 operator.go:125] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GatewayAPI GatewayAPIController HighlyAvailableArbiter ImageVolume IngressControllerLBSubnetsAWS KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController SigstoreImageVerification StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPClusterHostedDNSInstall GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall ImageModeStatusReporting ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesAzure ManagedBootImagesvSphere MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutatingAdmissionPolicy NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterOperations NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PreconfiguredUDNAddresses SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeAttributesClass VolumeGroupSnapshot] I0704 13:59:22.390344 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0704 13:59:22.390361 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0704 13:59:22.390371 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0704 13:59:22.390379 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0704 13:59:22.390390 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0704 13:59:22.390398 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0704 13:59:22.390888 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/tmp/serving-cert-407758658/tls.crt::/tmp/serving-cert-407758658/tls.key" I0704 13:59:22.391619 1 secure_serving.go:213] Serving securely on [::]:8443 I0704 13:59:22.391725 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0704 13:59:22.397740 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0704 13:59:22.397766 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0704 13:59:22.397881 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0704 13:59:22.406551 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 13:59:22.406574 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0704 13:59:22.413306 1 secretconfigobserver.go:119] support secret does not exist I0704 13:59:22.420262 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 13:59:22.426666 1 secretconfigobserver.go:119] support secret does not exist I0704 13:59:22.430349 1 recorder.go:161] Pruning old reports every 4h9m55s, max age is 288h0m0s I0704 13:59:22.439985 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0704 13:59:22.439986 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0704 13:59:22.440002 1 insightsreport.go:296] Starting report retriever I0704 13:59:22.440006 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0704 13:59:22.440027 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0704 13:59:22.440051 1 periodic.go:209] Running clusterconfig gatherer I0704 13:59:22.440096 1 tasks_processing.go:45] number of workers: 32 I0704 13:59:22.440118 1 tasks_processing.go:69] worker 1 listening for tasks. I0704 13:59:22.440132 1 tasks_processing.go:71] worker 1 working on container_images task. I0704 13:59:22.440133 1 tasks_processing.go:69] worker 0 listening for tasks. I0704 13:59:22.440138 1 tasks_processing.go:69] worker 16 listening for tasks. I0704 13:59:22.440141 1 tasks_processing.go:71] worker 0 working on feature_gates task. I0704 13:59:22.440142 1 tasks_processing.go:69] worker 7 listening for tasks. I0704 13:59:22.440152 1 tasks_processing.go:69] worker 24 listening for tasks. I0704 13:59:22.440159 1 tasks_processing.go:69] worker 8 listening for tasks. I0704 13:59:22.440159 1 tasks_processing.go:69] worker 2 listening for tasks. I0704 13:59:22.440164 1 tasks_processing.go:69] worker 17 listening for tasks. I0704 13:59:22.440169 1 tasks_processing.go:69] worker 5 listening for tasks. I0704 13:59:22.440174 1 tasks_processing.go:69] worker 9 listening for tasks. I0704 13:59:22.440170 1 tasks_processing.go:69] worker 18 listening for tasks. I0704 13:59:22.440176 1 tasks_processing.go:69] worker 4 listening for tasks. I0704 13:59:22.440185 1 tasks_processing.go:69] worker 20 listening for tasks. I0704 13:59:22.440182 1 tasks_processing.go:69] worker 6 listening for tasks. I0704 13:59:22.440192 1 tasks_processing.go:69] worker 21 listening for tasks. I0704 13:59:22.440178 1 tasks_processing.go:69] worker 19 listening for tasks. I0704 13:59:22.440187 1 tasks_processing.go:69] worker 10 listening for tasks. I0704 13:59:22.440194 1 tasks_processing.go:69] worker 11 listening for tasks. I0704 13:59:22.440203 1 tasks_processing.go:69] worker 12 listening for tasks. I0704 13:59:22.440198 1 tasks_processing.go:69] worker 22 listening for tasks. I0704 13:59:22.440204 1 tasks_processing.go:69] worker 3 listening for tasks. I0704 13:59:22.440206 1 tasks_processing.go:69] worker 23 listening for tasks. I0704 13:59:22.440212 1 tasks_processing.go:69] worker 13 listening for tasks. I0704 13:59:22.440216 1 tasks_processing.go:69] worker 28 listening for tasks. I0704 13:59:22.440218 1 tasks_processing.go:69] worker 14 listening for tasks. I0704 13:59:22.440221 1 tasks_processing.go:69] worker 15 listening for tasks. I0704 13:59:22.440218 1 tasks_processing.go:69] worker 26 listening for tasks. I0704 13:59:22.440225 1 tasks_processing.go:69] worker 25 listening for tasks. I0704 13:59:22.440225 1 tasks_processing.go:69] worker 27 listening for tasks. I0704 13:59:22.440229 1 tasks_processing.go:69] worker 30 listening for tasks. I0704 13:59:22.440233 1 tasks_processing.go:69] worker 29 listening for tasks. I0704 13:59:22.440233 1 tasks_processing.go:69] worker 31 listening for tasks. I0704 13:59:22.440306 1 tasks_processing.go:71] worker 31 working on operators_pods_and_events task. I0704 13:59:22.440367 1 tasks_processing.go:71] worker 10 working on oauths task. I0704 13:59:22.440375 1 tasks_processing.go:71] worker 20 working on service_accounts task. I0704 13:59:22.440381 1 tasks_processing.go:71] worker 22 working on authentication task. I0704 13:59:22.440387 1 tasks_processing.go:71] worker 16 working on lokistack task. I0704 13:59:22.440429 1 tasks_processing.go:71] worker 21 working on cluster_apiserver task. I0704 13:59:22.440378 1 tasks_processing.go:71] worker 5 working on image_pruners task. I0704 13:59:22.440368 1 tasks_processing.go:71] worker 28 working on storage_classes task. I0704 13:59:22.440625 1 tasks_processing.go:71] worker 17 working on dvo_metrics task. I0704 13:59:22.440673 1 tasks_processing.go:71] worker 9 working on version task. I0704 13:59:22.440377 1 tasks_processing.go:71] worker 12 working on olm_operators task. I0704 13:59:22.440608 1 tasks_processing.go:71] worker 7 working on schedulers task. I0704 13:59:22.440613 1 tasks_processing.go:71] worker 24 working on node_logs task. I0704 13:59:22.440617 1 tasks_processing.go:71] worker 8 working on machine_sets task. I0704 13:59:22.440614 1 tasks_processing.go:71] worker 6 working on machine_healthchecks task. I0704 13:59:22.440621 1 tasks_processing.go:71] worker 2 working on machine_config_pools task. I0704 13:59:22.440625 1 tasks_processing.go:71] worker 19 working on infrastructures task. I0704 13:59:22.440636 1 tasks_processing.go:71] worker 25 working on operators task. I0704 13:59:22.440639 1 tasks_processing.go:71] worker 14 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0704 13:59:22.440643 1 tasks_processing.go:71] worker 15 working on machine_autoscalers task. I0704 13:59:22.440647 1 tasks_processing.go:71] worker 26 working on monitoring_persistent_volumes task. I0704 13:59:22.440657 1 tasks_processing.go:71] worker 23 working on overlapping_namespace_uids task. I0704 13:59:22.440661 1 tasks_processing.go:71] worker 3 working on ingress task. I0704 13:59:22.440670 1 tasks_processing.go:71] worker 18 working on image_registries task. I0704 13:59:22.440684 1 tasks_processing.go:71] worker 13 working on cost_management_metrics_configs task. I0704 13:59:22.440373 1 tasks_processing.go:71] worker 11 working on ceph_cluster task. I0704 13:59:22.440690 1 tasks_processing.go:71] worker 30 working on active_alerts task. I0704 13:59:22.440695 1 tasks_processing.go:71] worker 27 working on validating_webhook_configurations task. W0704 13:59:22.443622 1 gather_active_alerts.go:54] Unable to load alerts client, no alerts will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0704 13:59:22.443661 1 tasks_processing.go:71] worker 30 working on sap_config task. I0704 13:59:22.443713 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 82.43µs to process 0 records I0704 13:59:22.440724 1 tasks_processing.go:71] worker 4 working on pod_network_connectivity_checks task. I0704 13:59:22.440734 1 tasks_processing.go:71] worker 29 working on sap_datahubs task. I0704 13:59:22.448576 1 tasks_processing.go:71] worker 8 working on config_maps task. I0704 13:59:22.448587 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 7.218208ms to process 0 records I0704 13:59:22.449016 1 tasks_processing.go:71] worker 16 working on machine_configs task. I0704 13:59:22.449025 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 8.616173ms to process 0 records I0704 13:59:22.449418 1 tasks_processing.go:71] worker 6 working on machines task. E0704 13:59:22.449434 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0704 13:59:22.449449 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 7.99876ms to process 0 records I0704 13:59:22.449605 1 tasks_processing.go:71] worker 22 working on jaegers task. I0704 13:59:22.449668 1 controller.go:128] Initializing last reported time to 0001-01-01T00:00:00Z I0704 13:59:22.449689 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0704 13:59:22.449701 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0704 13:59:22.449705 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0704 13:59:22.449724 1 controller.go:489] The operator is still being initialized I0704 13:59:22.449733 1 controller.go:512] The operator is healthy I0704 13:59:22.449841 1 recorder.go:75] Recording config/authentication with fingerprint=85a8f6ed7e06d20b30915799a68eb86c826dccf346a1d85649e1c19ef5727eec I0704 13:59:22.449857 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 9.209998ms to process 1 records I0704 13:59:22.453926 1 tasks_processing.go:71] worker 29 working on openstack_version task. I0704 13:59:22.453946 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 9.718662ms to process 0 records I0704 13:59:22.454076 1 tasks_processing.go:71] worker 10 working on ingress_certificates task. I0704 13:59:22.454214 1 recorder.go:75] Recording config/oauth with fingerprint=259ab26ad84c25f324d3b6438c7e72a85f0e1542c117535647c985ea0069f82c I0704 13:59:22.454231 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 13.567591ms to process 1 records I0704 13:59:22.454322 1 recorder.go:75] Recording config/featuregate with fingerprint=0c379b7d332d35988618c039736b27b71ce739dae03df81536aaab7c1bcdf566 I0704 13:59:22.454332 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 13.951808ms to process 1 records E0704 13:59:22.454337 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0704 13:59:22.454344 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 10.287634ms to process 0 records I0704 13:59:22.454346 1 tasks_processing.go:71] worker 0 working on openstack_controlplanes task. I0704 13:59:22.454360 1 tasks_processing.go:71] worker 15 working on container_runtime_configs task. I0704 13:59:22.454366 1 tasks_processing.go:71] worker 4 working on silenced_alerts task. W0704 13:59:22.454393 1 gather_silenced_alerts.go:38] Unable to load alerts client, no alerts will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0704 13:59:22.454350 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 12.025269ms to process 0 records I0704 13:59:22.454488 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 33.709µs to process 0 records I0704 13:59:22.454522 1 tasks_processing.go:71] worker 4 working on openshift_machine_api_events task. I0704 13:59:22.454603 1 tasks_processing.go:71] worker 21 working on proxies task. I0704 13:59:22.454657 1 recorder.go:75] Recording config/apiserver with fingerprint=475f43ac6a2e6b1924e8fa8f064be95a347921998806d2fe1c407bd03238bb50 I0704 13:59:22.454670 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 14.050208ms to process 1 records I0704 13:59:22.455447 1 tasks_processing.go:71] worker 30 working on sap_pods task. I0704 13:59:22.455456 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 11.733565ms to process 0 records I0704 13:59:22.455793 1 tasks_processing.go:71] worker 5 working on tsdb_status task. W0704 13:59:22.455822 1 gather_prometheus_tsdb_status.go:38] Unable to load metrics client, tsdb status cannot be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0704 13:59:22.456061 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=eddbab19bf68a269f498da8c3cfcbf314efa8892b7917542fc2146853b17b8b9 I0704 13:59:22.456076 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 15.229941ms to process 1 records I0704 13:59:22.456087 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 27.384µs to process 0 records I0704 13:59:22.456094 1 tasks_processing.go:71] worker 5 working on metrics task. W0704 13:59:22.456114 1 gather_most_recent_metrics.go:64] Unable to load metrics client, no metrics will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0704 13:59:22.456124 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 24.209µs to process 0 records I0704 13:59:22.456129 1 tasks_processing.go:71] worker 5 working on nodenetworkconfigurationpolicies task. I0704 13:59:22.456281 1 tasks_processing.go:71] worker 13 working on nodenetworkstates task. I0704 13:59:22.456293 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 13.055799ms to process 0 records I0704 13:59:22.456302 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 12.932205ms to process 0 records I0704 13:59:22.456307 1 tasks_processing.go:71] worker 11 working on pdbs task. I0704 13:59:22.458822 1 tasks_processing.go:71] worker 29 working on certificate_signing_requests task. I0704 13:59:22.458834 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 4.871465ms to process 0 records I0704 13:59:22.470740 1 tasks_processing.go:71] worker 22 working on install_plans task. I0704 13:59:22.470751 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 21.120661ms to process 0 records I0704 13:59:22.470849 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 16.39545ms to process 0 records I0704 13:59:22.470856 1 tasks_processing.go:71] worker 15 working on openstack_dataplanedeployments task. I0704 13:59:22.471193 1 tasks_processing.go:71] worker 3 working on aggregated_monitoring_cr_names task. I0704 13:59:22.471438 1 recorder.go:75] Recording config/ingress with fingerprint=09dd00b65da410ce02abef150c9be606ddc988401f2bfb9a4748626f3d234105 I0704 13:59:22.471481 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 28.415567ms to process 1 records E0704 13:59:22.471520 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0704 13:59:22.471533 1 gather.go:177] gatherer "clusterconfig" function "machines" took 21.920046ms to process 0 records I0704 13:59:22.471544 1 tasks_processing.go:71] worker 6 working on support_secret task. I0704 13:59:22.471556 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 15.260257ms to process 0 records I0704 13:59:22.471571 1 tasks_processing.go:71] worker 13 working on crds task. I0704 13:59:22.471576 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 15.417963ms to process 0 records I0704 13:59:22.471583 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 29.206234ms to process 0 records I0704 13:59:22.471586 1 tasks_processing.go:71] worker 5 working on openshift_logging task. I0704 13:59:22.471593 1 tasks_processing.go:71] worker 26 working on openstack_dataplanenodesets task. I0704 13:59:22.471761 1 tasks_processing.go:71] worker 2 working on qemu_kubevirt_launcher_logs task. I0704 13:59:22.471775 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 30.295832ms to process 0 records I0704 13:59:22.472292 1 tasks_processing.go:71] worker 21 working on networks task. I0704 13:59:22.472346 1 recorder.go:75] Recording config/proxy with fingerprint=d408a8ccb48072ba49d3478f247a9e01e3f9df1ee7a09e00ca04211399c81709 I0704 13:59:22.472362 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 17.667323ms to process 1 records I0704 13:59:22.472415 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 17.872318ms to process 0 records I0704 13:59:22.472441 1 tasks_processing.go:71] worker 4 working on image task. I0704 13:59:22.472616 1 tasks_processing.go:71] worker 19 working on clusterroles task. I0704 13:59:22.475923 1 recorder.go:75] Recording config/infrastructure with fingerprint=91b374d9574393c81e1389612674d1dd558ac483375889514b0f51f323ded034 I0704 13:59:22.475993 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 30.866589ms to process 1 records I0704 13:59:22.476245 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=a3a8bc3cfd985442c77b2f9d910d7261773e55654d01e9bcd135742f82da244a I0704 13:59:22.476371 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=16b69a5895f436d06d550ea282e0ec9aed1fe5f0a7b6ef8288a763d8d5ea1f60 I0704 13:59:22.476412 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 31.786575ms to process 2 records I0704 13:59:22.476435 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 16.969988ms to process 0 records I0704 13:59:22.476662 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=0dd0b79feb5d27700449bec4b65d06e8fcd6e604683f17b9d20bbc3d3f6705bd I0704 13:59:22.476674 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 31.427984ms to process 1 records I0704 13:59:22.476683 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 18.866207ms to process 0 records I0704 13:59:22.476751 1 tasks_processing.go:71] worker 30 working on mutating_webhook_configurations task. I0704 13:59:22.476833 1 tasks_processing.go:74] worker 0 stopped. I0704 13:59:22.476866 1 tasks_processing.go:71] worker 28 working on nodes task. I0704 13:59:22.477440 1 tasks_processing.go:74] worker 18 stopped. I0704 13:59:22.478331 1 tasks_processing.go:71] worker 7 working on storage_cluster task. I0704 13:59:22.478960 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=62b3f6661d7f916fe1cef5910a0a21c25003c8d39bfbb0ed4386ceb167030c33 I0704 13:59:22.478983 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 32.759301ms to process 1 records I0704 13:59:22.479264 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=a9af6d0f62a4c065da9f601213cdf531b036e0884c985aefe387e696eedcf2ad I0704 13:59:22.479357 1 tasks_processing.go:74] worker 27 stopped. I0704 13:59:22.479430 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=4b06fc5618826af5516410456bd3ad7820332d0c553ce8eac96e38c98c012349 I0704 13:59:22.479470 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=712ba8febb4fa51c69370ec99e3ad48b75fbb1f37963620821bcc741c4f5807d I0704 13:59:22.479556 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=ee5388d5ae85debc4311ce0185e2d59fda3ab8f9da6571946e72dd354de1d84d I0704 13:59:22.479612 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=fdefa5a2eb715161f7426f9b4ce54dc6833fbce8a49ca0734e6dbc6168c4377d I0704 13:59:22.479662 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=8897a4c2346c3199db0349882be8b479ad62eb99f67ddf0d9eee92c325ba785a I0704 13:59:22.479711 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=ed249795865fc692181716554fb0d2074378fad9233d0b952d2e7e7778b38726 I0704 13:59:22.479787 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=bce8d4805b015186e0aac3ff89a5c5358ee484b2c622212c88da447ef046ae02 I0704 13:59:22.479841 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=869f8070b720b5f919676a7766a298b7dbd01a0b3f7230b885d061a7cb98d828 I0704 13:59:22.479900 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=a6344fa3541d7222d03fba3c1047e0dd4afd22cb8ae4fe09599ecd4807b4b3de I0704 13:59:22.479954 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=2f7eb21343dd47b2850c44ce506709ef8b33d4a09175abc82a8142a545745155 I0704 13:59:22.479965 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 32.40901ms to process 11 records I0704 13:59:22.479990 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0704 13:59:22.479999 1 tasks_processing.go:74] worker 23 stopped. I0704 13:59:22.480000 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 36.729704ms to process 1 records I0704 13:59:22.481039 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0704 13:59:22.481090 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0704 13:59:22.481226 1 operator.go:288] started I0704 13:59:22.481258 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0704 13:59:22.481532 1 tasks_processing.go:74] worker 15 stopped. I0704 13:59:22.481571 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 10.655396ms to process 0 records I0704 13:59:22.482463 1 tasks_processing.go:74] worker 5 stopped. I0704 13:59:22.482474 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 10.862569ms to process 0 records W0704 13:59:22.483252 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0704 13:59:22.486008 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 7.451158ms to process 0 records I0704 13:59:22.486012 1 tasks_processing.go:74] worker 7 stopped. I0704 13:59:22.487605 1 tasks_processing.go:74] worker 1 stopped. I0704 13:59:22.488854 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-575cd97545-d8z4x with fingerprint=ea8397d99d953190bf8a85e3355fb5b2227ff88689a2dadec24779933aebfb0c I0704 13:59:22.489109 1 recorder.go:75] Recording config/pod/openshift-ovn-kubernetes/ovnkube-node-pp8vb with fingerprint=aa8a5fe72116bdac7e3fd48d6eacf0287d04e27699280f792b9bfaf310da3523 I0704 13:59:22.489162 1 recorder.go:75] Recording config/running_containers with fingerprint=13745dc1aded9dda0b2a6dd845f290efc6a3720b70e61d7678978d055352fd37 I0704 13:59:22.489172 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 47.459509ms to process 3 records I0704 13:59:22.490082 1 tasks_processing.go:74] worker 26 stopped. I0704 13:59:22.490094 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 18.481223ms to process 0 records I0704 13:59:22.490157 1 tasks_processing.go:74] worker 11 stopped. I0704 13:59:22.490262 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=a48d87a0d8c2f8f515a47339c1c33821c3d9d6471812ac7b19769d1e1554ee71 I0704 13:59:22.490292 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=b63d34dad51ec4beacb3d1b2b1eb080e432422f1197c8ec23e90b66f62673ab7 I0704 13:59:22.490332 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=36d47bb11f34f591ee3823bfc65263e5dda76af03eccbba161ca420d0001f2a5 I0704 13:59:22.490344 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 33.836873ms to process 3 records I0704 13:59:22.490489 1 tasks_processing.go:74] worker 4 stopped. I0704 13:59:22.490523 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0704 13:59:22.490532 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0704 13:59:22.490526 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0704 13:59:22.490623 1 recorder.go:75] Recording config/image with fingerprint=f9acd36e102308e20b9c3dc98dfce8b14e7147346f7edef961e2172b696b6bb0 I0704 13:59:22.490641 1 gather.go:177] gatherer "clusterconfig" function "image" took 17.955613ms to process 1 records I0704 13:59:22.491199 1 tasks_processing.go:74] worker 30 stopped. I0704 13:59:22.491379 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=b44e3cfffcad0b8a238976bdb1613dc52e39803170f6b8d41cfabfb354df2e66 I0704 13:59:22.491480 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=e314eb97e6ec92867aac1b8e9a2e57cf5f4152157ec90bc9a5dad20b26dd3fff I0704 13:59:22.491563 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=2e271a007f49d066651be8fe9c28772648d184cb43d81b177abdcefae90450be I0704 13:59:22.491614 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 14.413881ms to process 3 records I0704 13:59:22.491782 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0704 13:59:22.491799 1 controller.go:212] Source scaController *sca.Controller is not ready I0704 13:59:22.491805 1 controller.go:212] Source clusterTransferController *clustertransfer.Controller is not ready I0704 13:59:22.491810 1 recorder.go:75] Recording config/network with fingerprint=c7aa2ca2c44bfcede98b9e344772a1d46b62da9274f40995fc399869f1d12345 I0704 13:59:22.491811 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0704 13:59:22.491819 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0704 13:59:22.491821 1 gather.go:177] gatherer "clusterconfig" function "networks" took 18.98978ms to process 1 records I0704 13:59:22.491833 1 tasks_processing.go:74] worker 21 stopped. I0704 13:59:22.491845 1 controller.go:489] The operator is still being initialized I0704 13:59:22.491856 1 controller.go:512] The operator is healthy I0704 13:59:22.492301 1 tasks_processing.go:74] worker 6 stopped. E0704 13:59:22.492312 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0704 13:59:22.492321 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 20.749572ms to process 0 records I0704 13:59:22.492420 1 tasks_processing.go:74] worker 12 stopped. I0704 13:59:22.492480 1 recorder.go:75] Recording config/olm_operators with fingerprint=e506649afea136d6de69061ee71970fb5ee76201938f6d41e07df406f384b295 I0704 13:59:22.492522 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 51.287698ms to process 1 records I0704 13:59:22.492699 1 tasks_processing.go:74] worker 14 stopped. I0704 13:59:22.492717 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 50.71839ms to process 0 records I0704 13:59:22.492992 1 tasks_processing.go:74] worker 29 stopped. I0704 13:59:22.493011 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 34.157077ms to process 0 records I0704 13:59:22.497927 1 base_controller.go:82] Caches are synced for ConfigController I0704 13:59:22.497939 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0704 13:59:22.498751 1 prometheus_rules.go:88] Prometheus rules successfully created I0704 13:59:22.499685 1 tasks_processing.go:74] worker 19 stopped. I0704 13:59:22.500008 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=199e8f69d8593cdeb41ddfd7f2a027ed90ba4adfb4448a0d44045f3316df1b86 I0704 13:59:22.500176 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=6387de2f21efc051f7970de0665b720588daac989502b255f22d0ad3d179d522 I0704 13:59:22.500207 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 27.02546ms to process 2 records E0704 13:59:22.501748 1 cluster_transfer.go:95] failed to pull cluster transfer: unable to retrieve cluster transfer data from https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/: Get "https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/?search=cluster_uuid+is+%27052ce312-c6e7-42f2-b79b-a4d54fe821f4%27+and+status+is+%27accepted%27": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.0.15:53213->172.30.0.10:53: read: connection refused I0704 13:59:22.501762 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=Disconnected message=failed to pull cluster transfer: unable to retrieve cluster transfer data from https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/: Get "https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/?search=cluster_uuid+is+%27052ce312-c6e7-42f2-b79b-a4d54fe821f4%27+and+status+is+%27accepted%27": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.0.15:53213->172.30.0.10:53: read: connection refused I0704 13:59:22.502159 1 tasks_processing.go:74] worker 3 stopped. I0704 13:59:22.502179 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 30.94184ms to process 0 records I0704 13:59:22.502731 1 tasks_processing.go:74] worker 13 stopped. I0704 13:59:22.503248 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=d3a3899b871eee84c1e99538da72bab3ab45b3083861a23364ba5999f01ee37d I0704 13:59:22.503461 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=e41e972450887e328369c7375c486413720c3c5e64e68a07d35666cfa60e6eaf I0704 13:59:22.503475 1 gather.go:177] gatherer "clusterconfig" function "crds" took 31.144415ms to process 2 records I0704 13:59:22.504555 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 13:59:22.516018 1 tasks_processing.go:74] worker 8 stopped. E0704 13:59:22.516032 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0704 13:59:22.516038 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0704 13:59:22.516042 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0704 13:59:22.516050 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=26b6661162b099a0f5a279859b4f46c867929a79d9a4a41fde4be4e6fe138018 I0704 13:59:22.516097 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=f73fcd48d6f9ebbc83253a1b5f23ef59d0683465ec46e474389103c717467dfb I0704 13:59:22.516104 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0704 13:59:22.516109 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=c93090eb0d2a4736885abeb79c91680cfd01fda46464f83456b085d4dc8239f0 I0704 13:59:22.516113 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0704 13:59:22.516172 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0704 13:59:22.516180 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0704 13:59:22.516185 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 67.421805ms to process 7 records I0704 13:59:22.516322 1 tasks_processing.go:74] worker 10 stopped. E0704 13:59:22.516342 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0704 13:59:22.516352 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2rbbt77vn0u6or82g6ked39c37nbifmu-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2rbbt77vn0u6or82g6ked39c37nbifmu-primary-cert-bundle-secret" not found I0704 13:59:22.516413 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=13498b7a8d5b4939ee1c35e4d5f3ed957c50072a4a4deb6343f5593f7e27ad0d I0704 13:59:22.516425 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 62.22653ms to process 1 records I0704 13:59:22.526358 1 tasks_processing.go:74] worker 24 stopped. I0704 13:59:22.526375 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 85.02531ms to process 0 records I0704 13:59:22.527308 1 tasks_processing.go:74] worker 28 stopped. I0704 13:59:22.527612 1 recorder.go:75] Recording config/node/ip-10-0-0-7.ec2.internal with fingerprint=861c918ca1c77bf17864d45f365ad79f434698b9c9f4fdc1e8ae7e3fa2d1e47a I0704 13:59:22.527681 1 recorder.go:75] Recording config/node/ip-10-0-2-214.ec2.internal with fingerprint=0b247ef59ef3df4c47ba14ccdf4aac8bd03ae8c6afa9c5665b84987f300feb25 I0704 13:59:22.527691 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 50.392385ms to process 2 records I0704 13:59:22.532159 1 tasks_processing.go:74] worker 16 stopped. I0704 13:59:22.532188 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0704 13:59:22.532197 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 83.123547ms to process 1 records I0704 13:59:22.542246 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload W0704 13:59:22.546067 1 sca.go:161] Failed to pull SCA certs from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: unable to retrieve SCA certs data from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: Post "https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.0.15:45868->172.30.0.10:53: read: connection refused I0704 13:59:22.546083 1 controllerstatus.go:80] name=scaController healthy=true reason=NonHTTPError message=Failed to pull SCA certs from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: unable to retrieve SCA certs data from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: Post "https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.129.0.15:45868->172.30.0.10:53: read: connection refused I0704 13:59:22.551403 1 gather_logs.go:145] no pods in namespace were found I0704 13:59:22.551406 1 tasks_processing.go:74] worker 9 stopped. I0704 13:59:22.551708 1 recorder.go:75] Recording config/version with fingerprint=16e0d5a62f896fc6ce347c64ff0899c60813f3ce1fafbbff74b6b85cbcec0d27 I0704 13:59:22.551722 1 recorder.go:75] Recording config/id with fingerprint=0e21656628cb2750804c6d3e29d77e04f5e1bb1385f138630eba80b1ca7154bf I0704 13:59:22.551730 1 gather.go:177] gatherer "clusterconfig" function "version" took 110.362912ms to process 2 records I0704 13:59:22.551741 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 79.64261ms to process 0 records I0704 13:59:22.551747 1 tasks_processing.go:74] worker 2 stopped. I0704 13:59:22.581357 1 base_controller.go:82] Caches are synced for LoggingSyncer I0704 13:59:22.583278 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... W0704 13:59:23.478377 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0704 13:59:23.689346 1 gather_cluster_operator_pods_and_events.go:121] Found 16 pods with 19 containers I0704 13:59:23.689359 1 gather_cluster_operator_pods_and_events.go:235] Maximum buffer size: 1324517 bytes I0704 13:59:23.690164 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns container dns-default-jw22n pod in namespace openshift-dns (previous: false). I0704 13:59:23.910685 1 gather_cluster_operators.go:184] Unable to get configs.samples.operator.openshift.io resource due to: configs.samples.operator.openshift.io "cluster" not found I0704 13:59:23.916607 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-jw22n pod in namespace openshift-dns for failing operator dns (previous: false): "container \"dns\" in pod \"dns-default-jw22n\" is waiting to start: ContainerCreating" I0704 13:59:23.916625 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"dns\" in pod \"dns-default-jw22n\" is waiting to start: ContainerCreating" I0704 13:59:23.916635 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-rbac-proxy container dns-default-jw22n pod in namespace openshift-dns (previous: false). I0704 13:59:24.095059 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-jw22n pod in namespace openshift-dns for failing operator kube-rbac-proxy (previous: false): "container \"kube-rbac-proxy\" in pod \"dns-default-jw22n\" is waiting to start: ContainerCreating" I0704 13:59:24.095075 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"kube-rbac-proxy\" in pod \"dns-default-jw22n\" is waiting to start: ContainerCreating" I0704 13:59:24.095114 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns container dns-default-k6l7s pod in namespace openshift-dns (previous: false). I0704 13:59:24.320051 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-k6l7s pod in namespace openshift-dns for failing operator dns (previous: false): "container \"dns\" in pod \"dns-default-k6l7s\" is waiting to start: ContainerCreating" I0704 13:59:24.320068 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"dns\" in pod \"dns-default-k6l7s\" is waiting to start: ContainerCreating" I0704 13:59:24.320076 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-rbac-proxy container dns-default-k6l7s pod in namespace openshift-dns (previous: false). W0704 13:59:24.476685 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0704 13:59:24.494035 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-k6l7s pod in namespace openshift-dns for failing operator kube-rbac-proxy (previous: false): "container \"kube-rbac-proxy\" in pod \"dns-default-k6l7s\" is waiting to start: ContainerCreating" I0704 13:59:24.494056 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"kube-rbac-proxy\" in pod \"dns-default-k6l7s\" is waiting to start: ContainerCreating" I0704 13:59:24.494070 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-b4j66 pod in namespace openshift-dns (previous: false). I0704 13:59:24.513199 1 tasks_processing.go:74] worker 25 stopped. I0704 13:59:24.513247 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=8fa85c38fea0f359e14724af443bcabbb85b1e7035d45372e011d9d4e4b1ce9e I0704 13:59:24.513279 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=17ae893c328db0c232a969c12227b9fa2371cc5149e8875c89211ec472070355 I0704 13:59:24.513308 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0704 13:59:24.513334 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=14ca2d45a1e02225ea178b8d9c28f4a60c5e8a7597d1dc256689513e561186f7 I0704 13:59:24.513367 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0704 13:59:24.513391 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=b3fb154dc3302e9fac310ef902428e49349aa8236dcce9c9d2fe6cd818df301a I0704 13:59:24.513421 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=f9fb0531403dc7583a6e817a3884daefb48134ac8745b94de8496fb28233b803 I0704 13:59:24.513444 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=332192d14e2c077f29e5cb35c1cd41f942ac93a9b96e136ec1b6291799a0b636 I0704 13:59:24.513458 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=b6b769e5f61fd705b0af10ef69ef32e197e014db6fd54cb2aefd4bbed40c70cf I0704 13:59:24.513475 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=96c36df6f04c51ef7ffce82e5d525b0ecd1c572f2f0b0946ba169dcb84ff3330 I0704 13:59:24.513484 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0704 13:59:24.513527 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=91e6e661d27e98c9875e27288eaf5b848bd1651be2392c6e30c693138377d91a I0704 13:59:24.513541 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0704 13:59:24.513555 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=a16366d3a3524e4a70912579dfa2e43f317f519c992d5b669f8655f3b1a7f5a8 I0704 13:59:24.513565 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0704 13:59:24.513584 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=4a0e37aa6cce8a06990a8b2a6e7d570b300b697b8e99e7afc0a08fda66a46c11 I0704 13:59:24.513593 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0704 13:59:24.513608 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=2363ee6e0344fc5c06c37f2bab5131587a9e348dde03e360ac3e269429197124 I0704 13:59:24.513728 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=f74641f86bb7f378d8759a1f4cc63ae991d183c20c0f3fb65e2c0510c166f952 I0704 13:59:24.513737 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0704 13:59:24.513743 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0704 13:59:24.513764 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0704 13:59:24.513785 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=bace96c049979b49a0385b464e711e1648220e3b1aa553ad8362c8a56beb129e I0704 13:59:24.513808 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=8c67778864f97a416aedf917a17c5a492ecb43e76df62c445bb3ba93c2762f55 I0704 13:59:24.513817 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0704 13:59:24.513833 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=88541fcebe34ca6e603bbd7715b59b39134ef6e9f97dc3747ce1dc30f6533bc4 I0704 13:59:24.513842 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0704 13:59:24.513855 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=94a95a05c09687d96cf9c3b7c19dc1b53cbd0232dc1ebf84236970c03c83cd52 I0704 13:59:24.513869 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=9d2f1df8a4891e7a21311d458417f6e2e90eef5a2b345a8742f95a09b36e4273 I0704 13:59:24.513894 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=d9571b17d7e594a59149a6fac02938194d27ffe9e356ec262704b84d1a3bc71c I0704 13:59:24.513911 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=0af6b10cea6cd59ab214afd229ec41951b5d73a019c96a0fc9fdad48bfbaa9c9 I0704 13:59:24.513932 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=9b4c9d412f0c2af5a60e54cb828dde44a67053742ee2289431a2516fc638ddd9 I0704 13:59:24.513941 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0704 13:59:24.513968 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=468f2c5fe0163a60bb4d2c9382bb0a39fb7c1fc4741ac1c48ccfe83627390900 I0704 13:59:24.513983 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0704 13:59:24.513992 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0704 13:59:24.513998 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.071297009s to process 36 records I0704 13:59:24.694288 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0704 13:59:24.694305 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-lh22j pod in namespace openshift-dns (previous: false). I0704 13:59:24.894617 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0704 13:59:24.894704 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-76fd6f95b7-xsmt4 pod in namespace openshift-image-registry (previous: false). I0704 13:59:25.087520 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0704 13:59:25.087569 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-9997485b-5lzg4 pod in namespace openshift-image-registry (previous: false). I0704 13:59:25.295364 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for image-registry-9997485b-5lzg4 pod in namespace openshift-image-registry for failing operator registry (previous: false): "container \"registry\" in pod \"image-registry-9997485b-5lzg4\" is waiting to start: ContainerCreating" I0704 13:59:25.295381 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"registry\" in pod \"image-registry-9997485b-5lzg4\" is waiting to start: ContainerCreating" I0704 13:59:25.295414 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-9997485b-7sfpt pod in namespace openshift-image-registry (previous: false). W0704 13:59:25.476063 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0704 13:59:25.493045 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for image-registry-9997485b-7sfpt pod in namespace openshift-image-registry for failing operator registry (previous: false): "container \"registry\" in pod \"image-registry-9997485b-7sfpt\" is waiting to start: ContainerCreating" I0704 13:59:25.493061 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"registry\" in pod \"image-registry-9997485b-7sfpt\" is waiting to start: ContainerCreating" I0704 13:59:25.493071 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-46rxp pod in namespace openshift-image-registry (previous: false). I0704 13:59:25.694404 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0704 13:59:25.694425 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-grkx7 pod in namespace openshift-image-registry (previous: false). I0704 13:59:25.894051 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0704 13:59:25.894069 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-55469cdb6d-vgz8h pod in namespace openshift-ingress (previous: false). I0704 13:59:26.094019 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for router-default-55469cdb6d-vgz8h pod in namespace openshift-ingress for failing operator router (previous: false): "container \"router\" in pod \"router-default-55469cdb6d-vgz8h\" is waiting to start: ContainerCreating" I0704 13:59:26.094040 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"router\" in pod \"router-default-55469cdb6d-vgz8h\" is waiting to start: ContainerCreating" I0704 13:59:26.094056 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-55469cdb6d-xd89p pod in namespace openshift-ingress (previous: false). I0704 13:59:26.294236 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for router-default-55469cdb6d-xd89p pod in namespace openshift-ingress for failing operator router (previous: false): "container \"router\" in pod \"router-default-55469cdb6d-xd89p\" is waiting to start: ContainerCreating" I0704 13:59:26.294253 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"router\" in pod \"router-default-55469cdb6d-xd89p\" is waiting to start: ContainerCreating" I0704 13:59:26.294267 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-7479477965-8bkrj pod in namespace openshift-ingress (previous: false). W0704 13:59:26.476911 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0704 13:59:26.495010 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for router-default-7479477965-8bkrj pod in namespace openshift-ingress for failing operator router (previous: false): "container \"router\" in pod \"router-default-7479477965-8bkrj\" is waiting to start: ContainerCreating" I0704 13:59:26.495028 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"router\" in pod \"router-default-7479477965-8bkrj\" is waiting to start: ContainerCreating" I0704 13:59:26.495042 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-hc827 pod in namespace openshift-ingress-canary (previous: false). I0704 13:59:26.693923 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for ingress-canary-hc827 pod in namespace openshift-ingress-canary for failing operator serve-healthcheck-canary (previous: false): "container \"serve-healthcheck-canary\" in pod \"ingress-canary-hc827\" is waiting to start: ContainerCreating" I0704 13:59:26.693944 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"serve-healthcheck-canary\" in pod \"ingress-canary-hc827\" is waiting to start: ContainerCreating" I0704 13:59:26.693958 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-rzjd2 pod in namespace openshift-ingress-canary (previous: false). I0704 13:59:26.896386 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for ingress-canary-rzjd2 pod in namespace openshift-ingress-canary for failing operator serve-healthcheck-canary (previous: false): "container \"serve-healthcheck-canary\" in pod \"ingress-canary-rzjd2\" is waiting to start: ContainerCreating" I0704 13:59:26.896403 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"serve-healthcheck-canary\" in pod \"ingress-canary-rzjd2\" is waiting to start: ContainerCreating" I0704 13:59:26.896414 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for migrator container migrator-7d5f866c57-rrskq pod in namespace openshift-kube-storage-version-migrator (previous: false). I0704 13:59:27.115424 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for graceful-termination container migrator-7d5f866c57-rrskq pod in namespace openshift-kube-storage-version-migrator (previous: false). I0704 13:59:27.302015 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-storage-version-migrator-operator container kube-storage-version-migrator-operator-74848b4cb9-xlxlj pod in namespace openshift-kube-storage-version-migrator-operator (previous: false). W0704 13:59:27.473368 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0704 13:59:27.473396 1 tasks_processing.go:74] worker 17 stopped. E0704 13:59:27.473409 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0704 13:59:27.473422 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 W0704 13:59:27.473435 1 gather.go:155] issue recording gatherer "clusterconfig" function "dvo_metrics" result "config/dvo_metrics" because of the warning: warning: the record with the same fingerprint "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" was already recorded at path "config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt", recording another one with a different path "config/dvo_metrics" I0704 13:59:27.473445 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.032706326s to process 1 records I0704 13:59:27.518780 1 tasks_processing.go:74] worker 31 stopped. I0704 13:59:27.518880 1 recorder.go:75] Recording events/openshift-dns-operator with fingerprint=02b3ac864871a2e69e30523a7de19ca48dc264b15557123324e5b9ca252a902c I0704 13:59:27.518929 1 recorder.go:75] Recording events/openshift-dns with fingerprint=b7c0e65bd6fbee988c93c662695d5f954af2fdd7aad874a12ffe2c22572c3bfd I0704 13:59:27.519012 1 recorder.go:75] Recording events/openshift-image-registry with fingerprint=f3f704010f30b2b060ff768468a89839d43ee17e5ba5865682e8dab320b37c2a I0704 13:59:27.519043 1 recorder.go:75] Recording events/openshift-ingress-operator with fingerprint=ef1113f22acdc2a5e976a2a1b369cff1c2d75e55c4777a3f3788cccbface95f0 I0704 13:59:27.519088 1 recorder.go:75] Recording events/openshift-ingress with fingerprint=5e5479d2ba5398ea51084d1a010b28b9ad424ef5ecf8a2f14b5844ac5fc5d9cb I0704 13:59:27.519103 1 recorder.go:75] Recording events/openshift-ingress-canary with fingerprint=ae1b8497bb858770b4e0e4e6d3137fea074e5f6b8fe46d4b00b0c45da21e6c1c I0704 13:59:27.519115 1 recorder.go:75] Recording events/openshift-kube-storage-version-migrator with fingerprint=26e169ba3d85a041dac2d2005a52528309313d96ae3b50905437a73718e8ec01 I0704 13:59:27.519164 1 recorder.go:75] Recording events/openshift-kube-storage-version-migrator-operator with fingerprint=963d4895a78b59eebce9d3e07e143458d760618fe9abb36c89029a7ad27113d4 I0704 13:59:27.519303 1 recorder.go:75] Recording config/pod/openshift-image-registry/image-registry-76fd6f95b7-xsmt4 with fingerprint=8310f711f2d810149d6f13d5e8cd6dab0a9d4089e0a74078a84db2013da8dc73 I0704 13:59:27.519418 1 recorder.go:75] Recording config/pod/openshift-image-registry/image-registry-9997485b-5lzg4 with fingerprint=fea72b4ca881475b2e950fd7a44cbb7cc96cacf20ccd65f3eae760dc0bdb523a I0704 13:59:27.519531 1 recorder.go:75] Recording config/pod/openshift-image-registry/image-registry-9997485b-7sfpt with fingerprint=24c7eeaf07cc62f644c5ec127211393ec39b21b92e3136a5725c3284265b2f8f I0704 13:59:27.519547 1 recorder.go:75] Recording config/pod/openshift-kube-storage-version-migrator/logs/migrator-7d5f866c57-rrskq/migrator_current.log with fingerprint=d7e4cd6505787d31288dc1e0af271aa47ec05d84d79c695b3b3173b20282a794 I0704 13:59:27.519552 1 recorder.go:75] Recording config/pod/openshift-kube-storage-version-migrator/logs/migrator-7d5f866c57-rrskq/graceful-termination_current.log with fingerprint=f61c3fe0de9481045e469a287c28acc7dc6d94ec7c7eb0744841b48e88b3aab3 I0704 13:59:27.519624 1 recorder.go:75] Recording config/pod/openshift-kube-storage-version-migrator-operator/logs/kube-storage-version-migrator-operator-74848b4cb9-xlxlj/kube-storage-version-migrator-operator_current.log with fingerprint=98a403b331a66c29ec0c7c3fe739a4f066952b0d53a1b564e8ae8022dc49a607 I0704 13:59:27.519634 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 5.078452939s to process 14 records I0704 13:59:35.007285 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 13:59:35.568108 1 tasks_processing.go:74] worker 22 stopped. I0704 13:59:35.568152 1 recorder.go:75] Recording config/installplans with fingerprint=e3c18441424aaa81f54ea9723d194eacb5824eb47178047bd893de11825bc8a2 I0704 13:59:35.568166 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 13.097311124s to process 1 records I0704 13:59:36.153033 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 13:59:36.248997 1 tasks_processing.go:74] worker 20 stopped. I0704 13:59:36.249253 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=2795f4fb1eec9e9b515e9d465dc026684b73654f00c1120ebc66fca9dd8c1a2c I0704 13:59:36.249270 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.808604085s to process 1 records E0704 13:59:36.249341 1 periodic.go:247] "Unhandled Error" err="clusterconfig failed after 13.809s with: function \"machine_healthchecks\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machines\" failed with an error, function \"support_secret\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0704 13:59:36.250451 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "machine_healthchecks" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machines" failed with an error, function "support_secret" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0704 13:59:36.250464 1 periodic.go:209] Running workloads gatherer I0704 13:59:36.250477 1 tasks_processing.go:45] number of workers: 2 I0704 13:59:36.250482 1 tasks_processing.go:69] worker 1 listening for tasks. I0704 13:59:36.250486 1 tasks_processing.go:71] worker 1 working on workload_info task. I0704 13:59:36.250508 1 tasks_processing.go:69] worker 0 listening for tasks. I0704 13:59:36.250537 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0704 13:59:36.277114 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 20s for image data I0704 13:59:36.287478 1 gather_workloads_info.go:387] No image sha256:04c87c054a3f366a7dfbe0a93ebb0c80a098ee16842c5794b67c1202eec61996 (11ms) I0704 13:59:36.292422 1 tasks_processing.go:74] worker 0 stopped. I0704 13:59:36.292434 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 41.86536ms to process 0 records I0704 13:59:36.298080 1 gather_workloads_info.go:387] No image sha256:2904a78e2eb73fd6a9bb94c105c2a056831fb4113fbb7b0607c50adc9d879c9b (11ms) I0704 13:59:36.308035 1 gather_workloads_info.go:387] No image sha256:765f0d23b637f685f98a31bd47c131b03cf72a40761a3f9a9d6320faa3c33733 (10ms) I0704 13:59:36.317819 1 gather_workloads_info.go:387] No image sha256:0a99240166165eb5718e7516a43282fe32df9c7c5e809b31b58abe44e42ff94d (10ms) I0704 13:59:36.328145 1 gather_workloads_info.go:387] No image sha256:b3909bf664c77097f75b3768830863d642eed3815dab2bfb4415c771ca2d5007 (10ms) I0704 13:59:36.338632 1 gather_workloads_info.go:387] No image sha256:653c666f842c13e0baae2e89a9b1efe0e2ef56f621ffb5b32005115d2a26ab8c (10ms) I0704 13:59:36.348929 1 gather_workloads_info.go:387] No image sha256:521712486e2c6e3c020dad6a1cb340db8e55665b69f7c208fab9cd9e965fd588 (10ms) I0704 13:59:36.351111 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 13:59:36.359453 1 gather_workloads_info.go:387] No image sha256:56a85660a445eced5c79a595a0eccf590087c5672d50f49d4c25ad52f9a44f04 (11ms) I0704 13:59:36.369814 1 gather_workloads_info.go:387] No image sha256:e84cb128d930bd1ab867cc89b7b7bf2b2c0e41105ab93b5381069945b3ee9c57 (10ms) I0704 13:59:36.380661 1 gather_workloads_info.go:387] No image sha256:695cf2f0cc07683c2a3ce1eaf3e56fe18abc6e2bac716f7d9843f5d173b9df52 (11ms) I0704 13:59:36.390655 1 gather_workloads_info.go:387] No image sha256:ca1344cb64140188b7cae7bbc51fb751566c0b0c97d5e39b5850e628032c4a5e (10ms) I0704 13:59:36.489153 1 gather_workloads_info.go:387] No image sha256:a498046d64605bcccee2440aa4f04a4602baaae263cf01d977ec5208e876b1fd (98ms) I0704 13:59:36.588329 1 gather_workloads_info.go:387] No image sha256:a56211d075aa43cbb491f669a5b2e46ee023dc95b7d51dbac28f463948c5ad61 (99ms) I0704 13:59:36.688389 1 gather_workloads_info.go:387] No image sha256:2598489729a4b258e4ecda4a06f6875133f2a10ced5c5241f8a57a8a05418e36 (100ms) I0704 13:59:36.787486 1 gather_workloads_info.go:387] No image sha256:a258c226562adb14e3a163a1940938526ee6a0928982a7667d85d9a7334ce639 (99ms) I0704 13:59:36.887867 1 gather_workloads_info.go:387] No image sha256:88437b35173fe2e8a4d9f148ec17c754bee4df3f723b3a5676955eef4f363d31 (100ms) I0704 13:59:36.989679 1 gather_workloads_info.go:387] No image sha256:7adc1eab05d6724c76ba751f6df816b08d6e70b78dee9eb94fa6fd9690542c98 (102ms) I0704 13:59:37.087693 1 gather_workloads_info.go:387] No image sha256:2e57e192c3c1240fd935dcd55c8fde5e70e78bf81d6176c96edf21fafe59f8ba (98ms) I0704 13:59:37.188082 1 gather_workloads_info.go:387] No image sha256:289816958633a763a72dbc44e1dad40466223164e7e253039514f0d974ea5d21 (100ms) I0704 13:59:37.288118 1 gather_workloads_info.go:387] No image sha256:4556896f77307821531ef91b7b7faccb82b824ea695693b2989f597f0deca038 (100ms) I0704 13:59:37.388526 1 gather_workloads_info.go:387] No image sha256:2e564f336c77116053f34d4201d364d8da04e789cfffa0ea422574c95f2d6404 (100ms) I0704 13:59:37.487572 1 gather_workloads_info.go:387] No image sha256:943018739e3db1763c3184b460dbc409e058abbac76d57b9927faad317be85e4 (99ms) I0704 13:59:37.587697 1 gather_workloads_info.go:387] No image sha256:c940ea87e7d133d75ba0002ef00c0806825eed3db8094cdb260d1bac18127733 (100ms) I0704 13:59:37.687841 1 gather_workloads_info.go:387] No image sha256:5f0b67cfbbc381243fb91ccc17345b56d05f4d717c667e8c644e5bf05633ba71 (100ms) I0704 13:59:37.789016 1 gather_workloads_info.go:387] No image sha256:36b9e89c3cfcf1ab9ae500486e38afb6862cba48cb0b4d84a09508ab8f3d299f (101ms) I0704 13:59:37.888161 1 gather_workloads_info.go:387] No image sha256:91828234f107c068c8a4966d08370ae7b73e637651dbc6d92c18c4553402c22c (99ms) I0704 13:59:37.988260 1 gather_workloads_info.go:387] No image sha256:47154813651033d59751fb655a384dbffb64dd26f10bd7f3be0c3128d0486356 (100ms) I0704 13:59:38.088371 1 gather_workloads_info.go:387] No image sha256:1a2532940843248c57d52141185dd71fbc393ab28b65d48f682038632c1dbbad (100ms) I0704 13:59:38.188362 1 gather_workloads_info.go:387] No image sha256:5a95c19d82767e0235b4edb4a0536482c816904897aae1dc3eb255cb52b87a9f (100ms) I0704 13:59:38.288881 1 gather_workloads_info.go:387] No image sha256:a0105d1eb62cf6ac9e5e2ef28d3e89bf6dc514bc594fc7090fe5a5ee18a09c87 (100ms) I0704 13:59:38.388032 1 gather_workloads_info.go:387] No image sha256:95418d0ba6fc0aa96647d97cf6530967f7d7ae97a82ce7f7e41c8e44310b0b72 (99ms) I0704 13:59:38.488123 1 gather_workloads_info.go:387] No image sha256:7b31223098f08328f5ddea8e5b871dbbd5f5a61ec550e8956f66793c0c6031a9 (100ms) I0704 13:59:38.588631 1 gather_workloads_info.go:387] No image sha256:03cf4cd7ef1518610c6c7b3ad27d1622d82e98e3dc6e3f8e5d0fceb5c8d3786e (100ms) I0704 13:59:38.588663 1 tasks_processing.go:74] worker 1 stopped. E0704 13:59:38.588675 1 gather.go:140] gatherer "workloads" function "workload_info" failed with the error: no running pods found for the insights-runtime-extractor statefulset I0704 13:59:38.588985 1 recorder.go:75] Recording config/workload_info with fingerprint=c2f8440bb2e41329cffb5d64826b32b2476536dd19e529df565fa99e855041a7 I0704 13:59:38.589001 1 gather.go:177] gatherer "workloads" function "workload_info" took 2.338168431s to process 1 records E0704 13:59:38.589025 1 periodic.go:247] "Unhandled Error" err="workloads failed after 2.338s with: function \"workload_info\" failed with an error" I0704 13:59:38.590122 1 controllerstatus.go:89] name=periodic-workloads healthy=false reason=PeriodicGatherFailed message=Source workloads could not be retrieved: function "workload_info" failed with an error I0704 13:59:38.590135 1 periodic.go:209] Running conditional gatherer I0704 13:59:38.598352 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules I0704 13:59:38.604768 1 conditional_gatherer.go:107] Get "https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules": dial tcp: lookup console.redhat.com on 172.30.0.10:53: read udp 10.129.0.15:54090->172.30.0.10:53: read: connection refused E0704 13:59:38.605015 1 conditional_gatherer.go:322] unable to update alerts cache: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0704 13:59:38.605085 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0704 13:59:38.613367 1 conditional_gatherer.go:392] cluster version is '4.20.8' E0704 13:59:38.613380 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 13:59:38.613385 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 13:59:38.613389 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 13:59:38.613392 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 13:59:38.613397 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 13:59:38.613400 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 13:59:38.613403 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 13:59:38.613405 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 13:59:38.613408 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0704 13:59:38.613422 1 tasks_processing.go:45] number of workers: 3 I0704 13:59:38.613428 1 tasks_processing.go:69] worker 2 listening for tasks. I0704 13:59:38.613432 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0704 13:59:38.613441 1 tasks_processing.go:69] worker 0 listening for tasks. I0704 13:59:38.613456 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0704 13:59:38.613456 1 tasks_processing.go:69] worker 1 listening for tasks. I0704 13:59:38.613468 1 tasks_processing.go:74] worker 1 stopped. I0704 13:59:38.613475 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0704 13:59:38.613527 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=7034af97e7e41c22e4b775abdd4b9066c8ebb19da33eb7f69f39bfd2eb5f6406 I0704 13:59:38.613539 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 863ns to process 1 records I0704 13:59:38.613573 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=0394430c431eec4d48bb1811a90918e95161d2282c59af26f2473613cc0959db I0704 13:59:38.613580 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.717µs to process 1 records I0704 13:59:38.613586 1 tasks_processing.go:74] worker 0 stopped. I0704 13:59:38.613783 1 tasks_processing.go:74] worker 2 stopped. I0704 13:59:38.613799 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 287.38µs to process 0 records I0704 13:59:38.613833 1 controllerstatus.go:89] name=periodic-conditional healthy=false reason=NotAvailable message=Get "https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules": dial tcp: lookup console.redhat.com on 172.30.0.10:53: read udp 10.129.0.15:54090->172.30.0.10:53: read: connection refused I0704 13:59:38.613852 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=359de9c990c741675cec72fda96b5c3682221efdb4799f5eaa6e9805bcd3b5c1 I0704 13:59:38.640627 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=9bf59c12fbb0252c79d15f09a85f9f4740c10c64d8e6d0f914da123853fc411f I0704 13:59:38.640743 1 diskrecorder.go:70] Writing 110 records to /var/lib/insights-operator/insights-2026-07-04-135938.tar.gz I0704 13:59:38.647980 1 diskrecorder.go:51] Wrote 110 records to disk in 7ms I0704 13:59:38.648006 1 periodic.go:278] Gathering cluster info every 2h0m0s I0704 13:59:38.648022 1 periodic.go:279] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0704 13:59:47.896992 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 14:00:31.754879 1 observer_polling.go:111] Observed file "/var/run/secrets/serving-cert/tls.crt" has been created (hash="27cf587ba2f242fba724bda02ba39c1ce893a78e82c7d41b24b73479d61db33a") W0704 14:00:31.754910 1 builder.go:160] Restart triggered because of file /var/run/secrets/serving-cert/tls.crt was created I0704 14:00:31.754956 1 observer_polling.go:111] Observed file "/var/run/secrets/serving-cert/tls.key" has been created (hash="f31550b6a5e0edfe6d80dc1296494c893539d9d808f8f1dfddb3786cd60cf5b2") I0704 14:00:31.755065 1 genericapiserver.go:548] "[graceful-termination] shutdown event" name="ShutdownInitiated" I0704 14:00:31.755107 1 observer_polling.go:111] Observed file "/var/run/configmaps/service-ca-bundle/service-ca.crt" has been created (hash="51e3833892098a4d2a6634c4a5430dfdcc2363982e9434b1f71ae423c63b4dfa")