W0319 18:02:37.113267 1 cmd.go:257] Using insecure, self-signed certificates I0319 18:02:38.162559 1 start.go:138] Unable to read service ca bundle: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0319 18:02:38.162883 1 observer_polling.go:159] Starting file observer I0319 18:02:39.083111 1 operator.go:60] Starting insights-operator v0.0.0-master+$Format:%H$ I0319 18:02:39.083316 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0319 18:02:39.083883 1 secure_serving.go:57] Forcing use of http/1.1 only W0319 18:02:39.083912 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0319 18:02:39.083924 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0319 18:02:39.083931 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. I0319 18:02:39.083924 1 simple_featuregate_reader.go:171] Starting feature-gate-detector W0319 18:02:39.083935 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0319 18:02:39.083956 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0319 18:02:39.083960 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0319 18:02:39.089910 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"2b830ec1-413f-4033-ba05-6b47f9e26eb6", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}} I0319 18:02:39.089886 1 operator.go:125] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GatewayAPI GatewayAPIController HighlyAvailableArbiter ImageVolume IngressControllerLBSubnetsAWS KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController SigstoreImageVerification StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPClusterHostedDNSInstall GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall ImageModeStatusReporting ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesAzure ManagedBootImagesvSphere MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutatingAdmissionPolicy NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterOperations NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PreconfiguredUDNAddresses SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeAttributesClass VolumeGroupSnapshot] I0319 18:02:39.105570 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0319 18:02:39.105579 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0319 18:02:39.105580 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0319 18:02:39.105596 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0319 18:02:39.105598 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0319 18:02:39.105598 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0319 18:02:39.105907 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/tmp/serving-cert-1527433284/tls.crt::/tmp/serving-cert-1527433284/tls.key" I0319 18:02:39.106695 1 secure_serving.go:213] Serving securely on [::]:8443 I0319 18:02:39.106781 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0319 18:02:39.111276 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0319 18:02:39.111303 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0319 18:02:39.111476 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0319 18:02:39.136139 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0319 18:02:39.136199 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0319 18:02:39.148443 1 secretconfigobserver.go:119] support secret does not exist I0319 18:02:39.159926 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0319 18:02:39.169437 1 secretconfigobserver.go:119] support secret does not exist I0319 18:02:39.172039 1 recorder.go:161] Pruning old reports every 6h58m14s, max age is 288h0m0s I0319 18:02:39.180916 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0319 18:02:39.180938 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0319 18:02:39.180956 1 periodic.go:209] Running clusterconfig gatherer I0319 18:02:39.180956 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0319 18:02:39.180979 1 insightsreport.go:296] Starting report retriever I0319 18:02:39.180986 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0319 18:02:39.181022 1 tasks_processing.go:45] number of workers: 32 I0319 18:02:39.181049 1 tasks_processing.go:69] worker 3 listening for tasks. I0319 18:02:39.181059 1 tasks_processing.go:71] worker 3 working on cost_management_metrics_configs task. I0319 18:02:39.181063 1 tasks_processing.go:69] worker 10 listening for tasks. I0319 18:02:39.181070 1 tasks_processing.go:69] worker 6 listening for tasks. I0319 18:02:39.181073 1 tasks_processing.go:69] worker 12 listening for tasks. I0319 18:02:39.181081 1 tasks_processing.go:69] worker 9 listening for tasks. I0319 18:02:39.181087 1 tasks_processing.go:69] worker 14 listening for tasks. I0319 18:02:39.181088 1 tasks_processing.go:69] worker 8 listening for tasks. I0319 18:02:39.181096 1 tasks_processing.go:69] worker 23 listening for tasks. I0319 18:02:39.181067 1 tasks_processing.go:69] worker 11 listening for tasks. I0319 18:02:39.181099 1 tasks_processing.go:69] worker 7 listening for tasks. I0319 18:02:39.181082 1 tasks_processing.go:69] worker 13 listening for tasks. I0319 18:02:39.181110 1 tasks_processing.go:69] worker 18 listening for tasks. I0319 18:02:39.181108 1 tasks_processing.go:69] worker 5 listening for tasks. I0319 18:02:39.181109 1 tasks_processing.go:69] worker 25 listening for tasks. I0319 18:02:39.181103 1 tasks_processing.go:69] worker 24 listening for tasks. I0319 18:02:39.181103 1 tasks_processing.go:69] worker 22 listening for tasks. I0319 18:02:39.181102 1 tasks_processing.go:69] worker 4 listening for tasks. I0319 18:02:39.181115 1 tasks_processing.go:69] worker 19 listening for tasks. I0319 18:02:39.181128 1 tasks_processing.go:69] worker 0 listening for tasks. I0319 18:02:39.181130 1 tasks_processing.go:69] worker 21 listening for tasks. I0319 18:02:39.181132 1 tasks_processing.go:69] worker 16 listening for tasks. I0319 18:02:39.181138 1 tasks_processing.go:69] worker 30 listening for tasks. I0319 18:02:39.181136 1 tasks_processing.go:69] worker 29 listening for tasks. I0319 18:02:39.181140 1 tasks_processing.go:69] worker 17 listening for tasks. I0319 18:02:39.181145 1 tasks_processing.go:69] worker 2 listening for tasks. I0319 18:02:39.181145 1 tasks_processing.go:71] worker 10 working on nodenetworkconfigurationpolicies task. I0319 18:02:39.181150 1 tasks_processing.go:71] worker 8 working on sap_datahubs task. I0319 18:02:39.181152 1 tasks_processing.go:71] worker 6 working on container_images task. I0319 18:02:39.181165 1 tasks_processing.go:71] worker 11 working on clusterroles task. I0319 18:02:39.181174 1 tasks_processing.go:71] worker 25 working on overlapping_namespace_uids task. I0319 18:02:39.181117 1 tasks_processing.go:69] worker 26 listening for tasks. I0319 18:02:39.181312 1 tasks_processing.go:71] worker 26 working on qemu_kubevirt_launcher_logs task. I0319 18:02:39.181061 1 tasks_processing.go:69] worker 31 listening for tasks. I0319 18:02:39.181122 1 tasks_processing.go:69] worker 20 listening for tasks. I0319 18:02:39.181123 1 tasks_processing.go:69] worker 15 listening for tasks. I0319 18:02:39.181124 1 tasks_processing.go:69] worker 27 listening for tasks. I0319 18:02:39.181131 1 tasks_processing.go:69] worker 28 listening for tasks. I0319 18:02:39.181148 1 tasks_processing.go:71] worker 12 working on crds task. I0319 18:02:39.181153 1 tasks_processing.go:71] worker 7 working on openstack_dataplanenodesets task. I0319 18:02:39.181157 1 tasks_processing.go:71] worker 2 working on machines task. I0319 18:02:39.181161 1 tasks_processing.go:71] worker 13 working on openstack_version task. I0319 18:02:39.181911 1 tasks_processing.go:71] worker 28 working on tsdb_status task. W0319 18:02:39.182494 1 gather_prometheus_tsdb_status.go:38] Unable to load metrics client, tsdb status cannot be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0319 18:02:39.182510 1 tasks_processing.go:71] worker 28 working on jaegers task. I0319 18:02:39.181957 1 tasks_processing.go:71] worker 20 working on certificate_signing_requests task. I0319 18:02:39.182574 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 39.905µs to process 0 records I0319 18:02:39.181169 1 tasks_processing.go:71] worker 5 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0319 18:02:39.181230 1 tasks_processing.go:71] worker 9 working on mutating_webhook_configurations task. I0319 18:02:39.181233 1 tasks_processing.go:71] worker 0 working on openshift_logging task. I0319 18:02:39.181234 1 tasks_processing.go:71] worker 14 working on sap_config task. I0319 18:02:39.181260 1 tasks_processing.go:71] worker 24 working on openstack_controlplanes task. I0319 18:02:39.181264 1 tasks_processing.go:71] worker 4 working on nodes task. I0319 18:02:39.181267 1 tasks_processing.go:71] worker 19 working on networks task. I0319 18:02:39.181274 1 tasks_processing.go:71] worker 30 working on machine_autoscalers task. I0319 18:02:39.181277 1 tasks_processing.go:71] worker 16 working on active_alerts task. W0319 18:02:39.183498 1 gather_active_alerts.go:54] Unable to load alerts client, no alerts will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0319 18:02:39.183511 1 tasks_processing.go:71] worker 16 working on openstack_dataplanedeployments task. I0319 18:02:39.181277 1 tasks_processing.go:71] worker 21 working on cluster_apiserver task. I0319 18:02:39.183581 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 29.285µs to process 0 records I0319 18:02:39.181274 1 tasks_processing.go:71] worker 22 working on ingress_certificates task. I0319 18:02:39.181287 1 tasks_processing.go:71] worker 29 working on sap_pods task. I0319 18:02:39.181287 1 tasks_processing.go:71] worker 17 working on image_registries task. I0319 18:02:39.181120 1 tasks_processing.go:69] worker 1 listening for tasks. I0319 18:02:39.183782 1 tasks_processing.go:71] worker 1 working on image_pruners task. I0319 18:02:39.181159 1 tasks_processing.go:71] worker 23 working on machine_config_pools task. I0319 18:02:39.181966 1 tasks_processing.go:71] worker 31 working on machine_configs task. I0319 18:02:39.181977 1 tasks_processing.go:71] worker 27 working on ingress task. I0319 18:02:39.181165 1 tasks_processing.go:71] worker 18 working on openshift_machine_api_events task. I0319 18:02:39.181981 1 tasks_processing.go:71] worker 15 working on machine_healthchecks task. I0319 18:02:39.191342 1 tasks_processing.go:71] worker 10 working on infrastructures task. I0319 18:02:39.191355 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 10.181023ms to process 0 records I0319 18:02:39.192109 1 tasks_processing.go:71] worker 3 working on proxies task. I0319 18:02:39.192143 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 11.040668ms to process 0 records I0319 18:02:39.193598 1 tasks_processing.go:71] worker 0 working on feature_gates task. I0319 18:02:39.193654 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 10.744674ms to process 0 records I0319 18:02:39.193699 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 11.798029ms to process 0 records I0319 18:02:39.193732 1 tasks_processing.go:71] worker 13 working on node_logs task. I0319 18:02:39.193800 1 tasks_processing.go:71] worker 7 working on lokistack task. I0319 18:02:39.193809 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 12.285925ms to process 0 records E0319 18:02:39.193820 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0319 18:02:39.193828 1 gather.go:177] gatherer "clusterconfig" function "machines" took 12.199537ms to process 0 records I0319 18:02:39.193838 1 tasks_processing.go:71] worker 2 working on metrics task. W0319 18:02:39.193861 1 gather_most_recent_metrics.go:64] Unable to load metrics client, no metrics will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0319 18:02:39.193876 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 29.669µs to process 0 records I0319 18:02:39.193884 1 tasks_processing.go:71] worker 2 working on authentication task. I0319 18:02:39.194105 1 tasks_processing.go:71] worker 30 working on service_accounts task. I0319 18:02:39.194114 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 10.739605ms to process 0 records I0319 18:02:39.196320 1 tasks_processing.go:71] worker 24 working on pdbs task. I0319 18:02:39.196331 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 13.295498ms to process 0 records I0319 18:02:39.196351 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 13.379515ms to process 0 records I0319 18:02:39.196359 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 15.159136ms to process 0 records I0319 18:02:39.196365 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 13.833403ms to process 0 records I0319 18:02:39.196371 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 12.832306ms to process 0 records I0319 18:02:39.196379 1 tasks_processing.go:71] worker 16 working on silenced_alerts task. I0319 18:02:39.196398 1 tasks_processing.go:71] worker 8 working on config_maps task. W0319 18:02:39.196407 1 gather_silenced_alerts.go:38] Unable to load alerts client, no alerts will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0319 18:02:39.196416 1 tasks_processing.go:71] worker 16 working on validating_webhook_configurations task. I0319 18:02:39.196416 1 tasks_processing.go:71] worker 14 working on olm_operators task. I0319 18:02:39.196457 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 30.408µs to process 0 records I0319 18:02:39.196398 1 tasks_processing.go:71] worker 28 working on machine_sets task. I0319 18:02:39.201133 1 tasks_processing.go:71] worker 29 working on version task. I0319 18:02:39.201140 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 17.452483ms to process 0 records I0319 18:02:39.203972 1 tasks_processing.go:71] worker 15 working on support_secret task. E0319 18:02:39.203984 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0319 18:02:39.203998 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 19.871843ms to process 0 records I0319 18:02:39.205663 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0319 18:02:39.205668 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0319 18:02:39.205870 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0319 18:02:39.205887 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 12.062839ms to process 0 records I0319 18:02:39.205877 1 tasks_processing.go:71] worker 7 working on oauths task. I0319 18:02:39.206267 1 tasks_processing.go:71] worker 27 working on schedulers task. I0319 18:02:39.206562 1 recorder.go:75] Recording config/ingress with fingerprint=c3979b4df82478a0819623659530d2b07944f30d37bd5ee6463642a69cc65f4f I0319 18:02:39.206582 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 22.349435ms to process 1 records I0319 18:02:39.206732 1 tasks_processing.go:71] worker 13 working on dvo_metrics task. I0319 18:02:39.206817 1 controller.go:128] Initializing last reported time to 0001-01-01T00:00:00Z I0319 18:02:39.206878 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0319 18:02:39.206817 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 12.988545ms to process 0 records I0319 18:02:39.206905 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0319 18:02:39.206917 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0319 18:02:39.206922 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 10.282874ms to process 0 records I0319 18:02:39.206937 1 controller.go:489] The operator is still being initialized I0319 18:02:39.206946 1 controller.go:512] The operator is healthy I0319 18:02:39.207097 1 recorder.go:75] Recording config/apiserver with fingerprint=0d950b0b2a014867ecb6f3e43dab06e1e27d05a390a6a8f6cc69cb2f294cda12 I0319 18:02:39.207146 1 tasks_processing.go:71] worker 28 working on storage_classes task. I0319 18:02:39.207271 1 gather_logs.go:145] no pods in namespace were found I0319 18:02:39.207132 1 tasks_processing.go:71] worker 21 working on storage_cluster task. I0319 18:02:39.207158 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 23.367829ms to process 1 records I0319 18:02:39.207631 1 tasks_processing.go:71] worker 2 working on operators task. I0319 18:02:39.207821 1 recorder.go:75] Recording config/authentication with fingerprint=21fa6a390170b3833e9de447a62b0c08bcbfc02f69a893e4ff76483ce4d58f1f I0319 18:02:39.207880 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 13.287352ms to process 1 records I0319 18:02:39.207911 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 25.961626ms to process 0 records I0319 18:02:39.207922 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 23.553861ms to process 0 records I0319 18:02:39.208162 1 tasks_processing.go:71] worker 26 working on image task. I0319 18:02:39.208206 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=55e484b6ba3ed776d31c37591b0d3e3ef5eb8992078fae8ea1b7dc0c8fa33fce I0319 18:02:39.208275 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 23.854647ms to process 1 records I0319 18:02:39.208309 1 tasks_processing.go:71] worker 1 working on container_runtime_configs task. I0319 18:02:39.208277 1 tasks_processing.go:71] worker 18 working on ceph_cluster task. I0319 18:02:39.208477 1 tasks_processing.go:71] worker 10 working on monitoring_persistent_volumes task. I0319 18:02:39.209004 1 recorder.go:75] Recording config/infrastructure with fingerprint=ac4b864a7e080845a27b01432da88f585f690a76b5fec73ffdd871b17303e58a I0319 18:02:39.209021 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 16.51119ms to process 1 records I0319 18:02:39.209097 1 recorder.go:75] Recording config/proxy with fingerprint=8b965de9bca8e7c489c80a76f15b55fd32fa9d22c1b421b9a6247113f3c74516 I0319 18:02:39.209112 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 15.788797ms to process 1 records I0319 18:02:39.209100 1 tasks_processing.go:71] worker 3 working on operators_pods_and_events task. I0319 18:02:39.209190 1 tasks_processing.go:71] worker 0 working on aggregated_monitoring_cr_names task. I0319 18:02:39.209231 1 recorder.go:75] Recording config/featuregate with fingerprint=5ff7ecda901d651a2a4dda72bc36241ae76489b0609440b3d1fdc0c49d9369ec I0319 18:02:39.209269 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 14.302358ms to process 1 records I0319 18:02:39.209417 1 tasks_processing.go:71] worker 4 working on nodenetworkstates task. I0319 18:02:39.209649 1 recorder.go:75] Recording config/node/ip-10-0-2-231.ec2.internal with fingerprint=9407766c873c069c95b4325b5e255089f7c660bae7ce41faeb4c38af5358ef57 I0319 18:02:39.209664 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 24.980025ms to process 1 records I0319 18:02:39.209778 1 tasks_processing.go:71] worker 9 working on pod_network_connectivity_checks task. I0319 18:02:39.209864 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=2b3e217ae7da922209d37b8c7faf31c50b6fa482312dd3a9b027c7cf5d8fc595 I0319 18:02:39.209970 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=5f7fa9f67cdcbab4c073d4b31eb3aa6f078f9b656988fce25ff508339e72717c I0319 18:02:39.210045 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=192751849e73855b18342f78fffc8beb9f6682f89206aeceb378be73f68a7ab4 I0319 18:02:39.210086 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 25.389317ms to process 3 records I0319 18:02:39.210116 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 24.570091ms to process 0 records I0319 18:02:39.210315 1 recorder.go:75] Recording config/network with fingerprint=48581f734e0c1437c83966451f37ffb208c1f0a81c33bea46395bb19bc2395f1 I0319 18:02:39.210329 1 gather.go:177] gatherer "clusterconfig" function "networks" took 25.447194ms to process 1 records I0319 18:02:39.210338 1 tasks_processing.go:71] worker 19 working on install_plans task. I0319 18:02:39.210374 1 tasks_processing.go:74] worker 23 stopped. I0319 18:02:39.212109 1 tasks_processing.go:74] worker 20 stopped. I0319 18:02:39.212173 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 29.548219ms to process 0 records I0319 18:02:39.212571 1 tasks_processing.go:74] worker 25 stopped. I0319 18:02:39.212602 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0319 18:02:39.212620 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 31.386714ms to process 1 records I0319 18:02:39.213786 1 base_controller.go:82] Caches are synced for ConfigController I0319 18:02:39.214581 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0319 18:02:39.214007 1 tasks_processing.go:74] worker 14 stopped. I0319 18:02:39.214017 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 17.577304ms to process 0 records I0319 18:02:39.214758 1 tasks_processing.go:74] worker 17 stopped. I0319 18:02:39.215158 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=9bece9dbe1fd4661c7e408e92d9d1ad901edd29f4a5cbcae17f54d2ad4dd3171 I0319 18:02:39.215178 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 30.323091ms to process 1 records I0319 18:02:39.217394 1 tasks_processing.go:74] worker 6 stopped. I0319 18:02:39.218656 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-575cd97545-k9fql with fingerprint=2e6eb67e6826b289db801b56f3312fa0f8be6b3e0379c57005dd9c5a401874e9 I0319 18:02:39.218728 1 recorder.go:75] Recording config/running_containers with fingerprint=508f68a3a3b87a6b3ecebbefe09b3bc4c4d8c027f1bae94c42ba79b7bb8094d5 I0319 18:02:39.218743 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 36.212068ms to process 2 records I0319 18:02:39.218822 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=8a8dec61b699712fa57fa04fb036b38cf8f85db15141224a0761b8c756870455 I0319 18:02:39.218824 1 tasks_processing.go:74] worker 24 stopped. I0319 18:02:39.218849 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=be38ae5fbc5110c9e457c409e85afc7d1f50c44024961fb5ac667180fd1d833d I0319 18:02:39.218876 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=37955e6f3d9615309dfd5f6541ec98d9088549e959197eff89011cec1b478661 I0319 18:02:39.218885 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 21.303205ms to process 3 records I0319 18:02:39.218964 1 tasks_processing.go:74] worker 11 stopped. I0319 18:02:39.219045 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=125fb1a8b780dd15418b66c0ee157852eaddcdea1d0d58ac7f2834f408a07a5e I0319 18:02:39.219135 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=f04d8cd8fb1c617e60c0e7d15ba2ce8e053a2354143d686493374fd7f91cfb5b I0319 18:02:39.219146 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 36.784073ms to process 2 records E0319 18:02:39.219160 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0319 18:02:39.219171 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 14.77535ms to process 0 records I0319 18:02:39.219179 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 10.591126ms to process 0 records I0319 18:02:39.219224 1 tasks_processing.go:74] worker 15 stopped. I0319 18:02:39.219253 1 tasks_processing.go:74] worker 1 stopped. I0319 18:02:39.219254 1 tasks_processing.go:74] worker 7 stopped. I0319 18:02:39.219388 1 recorder.go:75] Recording config/oauth with fingerprint=36967f10532702c64ad0579ae6cf289610fb1124faad5870342b9395e3747e4d I0319 18:02:39.219401 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 13.052949ms to process 1 records I0319 18:02:39.219577 1 tasks_processing.go:74] worker 18 stopped. I0319 18:02:39.219586 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 11.157394ms to process 0 records I0319 18:02:39.220430 1 tasks_processing.go:74] worker 16 stopped. I0319 18:02:39.220531 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=9c2ceb352ae19fe93e27bc9aff11887f367b7b1422fc24bd3de48719b47366d1 I0319 18:02:39.220603 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=fe126ae6e438abd16d7ad1fbeb0018b29c6c22b80977540a77019858c10fc678 I0319 18:02:39.220622 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=4b89250e42e508fe3cd1022e02091134233a929dd1d76668ca2675bf8d4c380f I0319 18:02:39.220645 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=2ffb00688bd2318a607e8c7756b8dfd79390d52bf876153bd484e1c3394e3233 I0319 18:02:39.220672 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=b2d107bdc6498442c5ed4b9d80fdea71a3e4e617a49c57667f2bc8e4597ee82b I0319 18:02:39.220695 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=de8020bd8eaf5090009e251a8fe7037e372033defad73a525de9c424effb412e I0319 18:02:39.220718 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=201a3f36212c2c54f0fb6b2bd13d4037fe8ae61c3c4a3cf85bccc1f1c14dd2e9 I0319 18:02:39.220776 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=df48e7d2b35e00ec75db5c4353c38823574d015347dd9eaf71c20534252db9ad I0319 18:02:39.220799 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=787502346126a2061fdeff1587162bb6f450e3f7501de274ca69a343b541cc01 I0319 18:02:39.220821 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=d6a9d9c51ea5738ea0700221ad86ae7f245b6f6feada229726a4800cef8765a3 I0319 18:02:39.220845 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=5406eaffd1d05c03209abdc546d857fc0e243ef7191c9987936c083e08f01ea8 I0319 18:02:39.220853 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 24.001385ms to process 11 records I0319 18:02:39.220918 1 tasks_processing.go:74] worker 21 stopped. I0319 18:02:39.220938 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 13.557463ms to process 0 records I0319 18:02:39.221113 1 tasks_processing.go:74] worker 27 stopped. I0319 18:02:39.221197 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=e195aa2513235b7fd567f586ac1466106fb2521f81111a3910aa50fff04c0014 I0319 18:02:39.221212 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 14.819622ms to process 1 records I0319 18:02:39.221893 1 tasks_processing.go:74] worker 4 stopped. I0319 18:02:39.221908 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 12.458516ms to process 0 records I0319 18:02:39.222359 1 tasks_processing.go:74] worker 9 stopped. E0319 18:02:39.222370 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0319 18:02:39.222377 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 12.563363ms to process 0 records I0319 18:02:39.222776 1 tasks_processing.go:74] worker 5 stopped. I0319 18:02:39.222794 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 40.053559ms to process 0 records W0319 18:02:39.226177 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0319 18:02:39.234752 1 configmapobserver.go:84] configmaps "insights-config" not found I0319 18:02:39.235010 1 tasks_processing.go:74] worker 12 stopped. I0319 18:02:39.235222 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0319 18:02:39.235260 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s I0319 18:02:39.235404 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer W0319 18:02:39.235388 1 operator.go:288] started I0319 18:02:39.235758 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=a7c54f3813e2b45489e4aa8b215ee7acdac25291b0609dc316a1bcbabbbbbd78 I0319 18:02:39.236071 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=87644e7d51bcb5d81f1990b24ea2755586887663d0448855d4631d84fe214c1a I0319 18:02:39.236089 1 gather.go:177] gatherer "clusterconfig" function "crds" took 53.581609ms to process 2 records I0319 18:02:39.236761 1 tasks_processing.go:74] worker 28 stopped. I0319 18:02:39.236870 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=1c54811071e42d494210fc1896baf43c5cc25822631f46c226175a47654937e0 I0319 18:02:39.236895 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=d33f998815e518c6dc59bbb7c9f406637dc277d21d1c457147044d9ec0aae807 I0319 18:02:39.236909 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 29.568389ms to process 2 records I0319 18:02:39.236922 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 28.273371ms to process 0 records I0319 18:02:39.236930 1 tasks_processing.go:74] worker 10 stopped. I0319 18:02:39.237761 1 tasks_processing.go:74] worker 26 stopped. I0319 18:02:39.237832 1 recorder.go:75] Recording config/image with fingerprint=a6b293ea137ea6a1b8563e9667228259942aeb90a94737b807fd2e3cdf6e59cf I0319 18:02:39.237843 1 gather.go:177] gatherer "clusterconfig" function "image" took 29.546287ms to process 1 records I0319 18:02:39.244267 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0319 18:02:39.244279 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0319 18:02:39.244284 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0319 18:02:39.244289 1 controller.go:212] Source scaController *sca.Controller is not ready I0319 18:02:39.244293 1 controller.go:212] Source clusterTransferController *clustertransfer.Controller is not ready I0319 18:02:39.244316 1 controller.go:489] The operator is still being initialized I0319 18:02:39.244326 1 controller.go:512] The operator is healthy I0319 18:02:39.248500 1 prometheus_rules.go:88] Prometheus rules successfully created I0319 18:02:39.254151 1 tasks_processing.go:74] worker 0 stopped. I0319 18:02:39.254169 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 44.940447ms to process 0 records E0319 18:02:39.254501 1 cluster_transfer.go:95] failed to pull cluster transfer: unable to retrieve cluster transfer data from https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/: Get "https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/?search=cluster_uuid+is+%27bd10ef77-1777-408e-9575-b11f4f3f530f%27+and+status+is+%27accepted%27": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.18:51209->172.30.0.10:53: read: connection refused I0319 18:02:39.254515 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=Disconnected message=failed to pull cluster transfer: unable to retrieve cluster transfer data from https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/: Get "https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/?search=cluster_uuid+is+%27bd10ef77-1777-408e-9575-b11f4f3f530f%27+and+status+is+%27accepted%27": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.18:51209->172.30.0.10:53: read: connection refused I0319 18:02:39.259708 1 tasks_processing.go:74] worker 8 stopped. E0319 18:02:39.259726 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0319 18:02:39.259735 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0319 18:02:39.259742 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0319 18:02:39.259755 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=26b6661162b099a0f5a279859b4f46c867929a79d9a4a41fde4be4e6fe138018 I0319 18:02:39.259799 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0319 18:02:39.259808 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0319 18:02:39.259814 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=c93090eb0d2a4736885abeb79c91680cfd01fda46464f83456b085d4dc8239f0 I0319 18:02:39.259820 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0319 18:02:39.260085 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0319 18:02:39.260156 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0319 18:02:39.260187 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 63.289532ms to process 7 records I0319 18:02:39.285648 1 tasks_processing.go:74] worker 31 stopped. I0319 18:02:39.285691 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0319 18:02:39.285709 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 101.797299ms to process 1 records I0319 18:02:39.295010 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0319 18:02:39.298704 1 tasks_processing.go:74] worker 29 stopped. W0319 18:02:39.298935 1 sca.go:161] Failed to pull SCA certs from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: unable to retrieve SCA certs data from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: Post "https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.18:55887->172.30.0.10:53: read: connection refused I0319 18:02:39.298953 1 controllerstatus.go:80] name=scaController healthy=true reason=NonHTTPError message=Failed to pull SCA certs from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: unable to retrieve SCA certs data from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: Post "https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.18:55887->172.30.0.10:53: read: connection refused I0319 18:02:39.298959 1 recorder.go:75] Recording config/version with fingerprint=ffb61879d80b56b2d4f34dd5900e35807f8d852c08404815d85bd535a92051ae I0319 18:02:39.298971 1 recorder.go:75] Recording config/id with fingerprint=faf7c83cf1e464876b91a9daca1a91034dcc67d46188e8d0c30525107fdd4733 I0319 18:02:39.298977 1 gather.go:177] gatherer "clusterconfig" function "version" took 97.547944ms to process 2 records I0319 18:02:39.335488 1 base_controller.go:82] Caches are synced for LoggingSyncer I0319 18:02:39.335506 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0319 18:02:39.336175 1 tasks_processing.go:74] worker 22 stopped. E0319 18:02:39.336193 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0319 18:02:39.336201 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2p4uhd769gqvcr0p2ip1o0n27iac050h-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2p4uhd769gqvcr0p2ip1o0n27iac050h-primary-cert-bundle-secret" not found I0319 18:02:39.336286 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=4f65f0a5ef895c0d93982aa35e046df6b4f101cb5fea1431844ab1c39f0ab05a I0319 18:02:39.336300 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 152.5067ms to process 1 records W0319 18:02:40.224997 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0319 18:02:40.460065 1 gather_cluster_operator_pods_and_events.go:121] Found 12 pods with 14 containers I0319 18:02:40.460080 1 gather_cluster_operator_pods_and_events.go:235] Maximum buffer size: 1797558 bytes I0319 18:02:40.460465 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns container dns-default-ghlcd pod in namespace openshift-dns (previous: false). I0319 18:02:40.656234 1 gather_cluster_operators.go:184] Unable to get configs.samples.operator.openshift.io resource due to: configs.samples.operator.openshift.io "cluster" not found I0319 18:02:40.680689 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-ghlcd pod in namespace openshift-dns for failing operator dns (previous: false): "container \"dns\" in pod \"dns-default-ghlcd\" is waiting to start: ContainerCreating" I0319 18:02:40.680710 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"dns\" in pod \"dns-default-ghlcd\" is waiting to start: ContainerCreating" I0319 18:02:40.680717 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-rbac-proxy container dns-default-ghlcd pod in namespace openshift-dns (previous: false). I0319 18:02:40.855630 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-ghlcd pod in namespace openshift-dns for failing operator kube-rbac-proxy (previous: false): "container \"kube-rbac-proxy\" in pod \"dns-default-ghlcd\" is waiting to start: ContainerCreating" I0319 18:02:40.855650 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"kube-rbac-proxy\" in pod \"dns-default-ghlcd\" is waiting to start: ContainerCreating" I0319 18:02:40.855660 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-jf2jl pod in namespace openshift-dns (previous: false). I0319 18:02:41.053171 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0319 18:02:41.053192 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-656b7bc594-b74nm pod in namespace openshift-image-registry (previous: false). I0319 18:02:41.054700 1 tasks_processing.go:74] worker 2 stopped. I0319 18:02:41.054748 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=02b158346c97e93e844b6c8822eeddd2d72d715a2ae9cc39a1566d18c8c56184 I0319 18:02:41.054789 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=ea2d195c99cb45a881eed5961caee5ed427a4b9513da2c979b404ca6449d6323 I0319 18:02:41.054819 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0319 18:02:41.054846 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=32262ee9ca10ea1250b66ebc6e7fe5ab8fb7a2311df766a412fe7acac97ccb5a I0319 18:02:41.054863 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0319 18:02:41.054886 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=67af1ba6eea38cee828742fcf54d961d44e7ac6197951bf24b9dc7cd260fa77a I0319 18:02:41.054920 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=fe8ef7bc4db5e4dd36cc4346d57a8a62b46a041803173f467d79b0b453741af3 I0319 18:02:41.054959 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=081f4582c8291d471230ca10170ade245d8096c0589d2d9fe009d9f6c725c071 I0319 18:02:41.054974 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=58356f065ae4044e50eb50c2b784669f0f868a63d8da9cff4a8129bdda6f9255 I0319 18:02:41.054991 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=f670d5a54063c4da24f82e24c35282817e3a184406a4fe1a9ff5c5099d45e49c I0319 18:02:41.055001 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0319 18:02:41.055016 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=a841331d4364f4dc8a567795782d6df1ef3c2d7571a4f7f467b0ce325e6d9647 I0319 18:02:41.055027 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0319 18:02:41.055042 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=43a3728f437e63495b33fdbd557abe25c19dd6465ffd0c24229b8f24bc47affb I0319 18:02:41.055051 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0319 18:02:41.055070 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=a474d06d4776ac7c3a6d57bead76b9e54d9193b1c02f70ee813935a6b1784b18 I0319 18:02:41.055079 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0319 18:02:41.055093 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=bbf8d0327e51ab79e1cb9ecfbcd1ac128374aa599ffa37c2377acd74a8f02462 I0319 18:02:41.055206 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=86abdc86d0c5105d4236b2d510331017340f02007d6bec3bd676ae8bfccedc3f I0319 18:02:41.055216 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0319 18:02:41.055222 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0319 18:02:41.055258 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0319 18:02:41.055281 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=6d0e2c790a56254b50d7a2d062c39cc4e97cd8ad67999a24efc25bac190731d1 I0319 18:02:41.055306 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=aa5f385d4d6d00d0b5af21b5a07f23dfd92397906c03fe0096710e22e40d91a3 I0319 18:02:41.055316 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0319 18:02:41.055331 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=e252cb837f741d2b61273e9137ab972f9527dbdfbf8e91e7bdfd518238baa934 I0319 18:02:41.055340 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0319 18:02:41.055353 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=3ab45b937e877359b410bdd9aa80c13b2d4ee316b1a1084eae2fd8e97bd607c3 I0319 18:02:41.055367 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=dd74b51137399bf1f58c85e5d56102e7ec60ad81d3fa3c1d08dd122725d32dfd I0319 18:02:41.055383 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=af2dd9db691cd8f38d5b7890c13c6c1c50e6e09dd827293dedcd91c1d1f4831d I0319 18:02:41.055398 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=d1261cb0a4b47322d789b8c0b50ea67c6809539500ab60ec86c640f326a03e58 I0319 18:02:41.055412 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=e1994a9691926f5d78ead46680ac83d1df5d0f2bd2ba590420becaeb47861d03 I0319 18:02:41.055435 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=733077e1a9f70d85d18dcafa06acc4049ed9331b304822e27f6a6f38490cbedb I0319 18:02:41.055451 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0319 18:02:41.055457 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0319 18:02:41.055463 1 gather.go:177] gatherer "clusterconfig" function "operators" took 1.84701148s to process 35 records W0319 18:02:41.225164 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0319 18:02:41.245999 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0319 18:02:41.246017 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-656b7bc594-cw7f5 pod in namespace openshift-image-registry (previous: false). I0319 18:02:41.456381 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for image-registry-656b7bc594-cw7f5 pod in namespace openshift-image-registry for failing operator registry (previous: false): "container \"registry\" in pod \"image-registry-656b7bc594-cw7f5\" is waiting to start: ContainerCreating" I0319 18:02:41.456405 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"registry\" in pod \"image-registry-656b7bc594-cw7f5\" is waiting to start: ContainerCreating" I0319 18:02:41.456419 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-6c58f67c85-nbzkg pod in namespace openshift-image-registry (previous: false). I0319 18:02:41.645932 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0319 18:02:41.645954 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-vnxt9 pod in namespace openshift-image-registry (previous: false). I0319 18:02:41.853024 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0319 18:02:41.853049 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-85665b79d9-ggfss pod in namespace openshift-ingress (previous: false). I0319 18:02:42.052304 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for router-default-85665b79d9-ggfss pod in namespace openshift-ingress for failing operator router (previous: false): "container \"router\" in pod \"router-default-85665b79d9-ggfss\" is waiting to start: ContainerCreating" I0319 18:02:42.052327 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"router\" in pod \"router-default-85665b79d9-ggfss\" is waiting to start: ContainerCreating" I0319 18:02:42.052338 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-86f77d84cc-4v6sc pod in namespace openshift-ingress (previous: false). W0319 18:02:42.225409 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0319 18:02:42.245965 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0319 18:02:42.245985 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-86f77d84cc-jxghd pod in namespace openshift-ingress (previous: false). I0319 18:02:42.456212 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for router-default-86f77d84cc-jxghd pod in namespace openshift-ingress for failing operator router (previous: false): "container \"router\" in pod \"router-default-86f77d84cc-jxghd\" is waiting to start: ContainerCreating" I0319 18:02:42.456231 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"router\" in pod \"router-default-86f77d84cc-jxghd\" is waiting to start: ContainerCreating" I0319 18:02:42.456269 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-2bf9r pod in namespace openshift-ingress-canary (previous: false). I0319 18:02:42.652524 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for ingress-canary-2bf9r pod in namespace openshift-ingress-canary for failing operator serve-healthcheck-canary (previous: false): "container \"serve-healthcheck-canary\" in pod \"ingress-canary-2bf9r\" is waiting to start: ContainerCreating" I0319 18:02:42.652548 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"serve-healthcheck-canary\" in pod \"ingress-canary-2bf9r\" is waiting to start: ContainerCreating" I0319 18:02:42.652561 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for migrator container migrator-7d5f866c57-n46dq pod in namespace openshift-kube-storage-version-migrator (previous: false). I0319 18:02:42.863456 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for graceful-termination container migrator-7d5f866c57-n46dq pod in namespace openshift-kube-storage-version-migrator (previous: false). I0319 18:02:43.053495 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-storage-version-migrator-operator container kube-storage-version-migrator-operator-74848b4cb9-8cw7f pod in namespace openshift-kube-storage-version-migrator-operator (previous: false). W0319 18:02:43.224897 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0319 18:02:43.257887 1 tasks_processing.go:74] worker 3 stopped. I0319 18:02:43.257977 1 recorder.go:75] Recording events/openshift-dns with fingerprint=0a281dd5f32c177f4f00d56b63713bfe77c937213bf325fb19dadf60813e63e5 I0319 18:02:43.258047 1 recorder.go:75] Recording events/openshift-image-registry with fingerprint=079fa4ca6e93647370874bd31e3480af54f6ce11aa395a9d4aa4b0660da89efa I0319 18:02:43.258074 1 recorder.go:75] Recording events/openshift-ingress-operator with fingerprint=29a4ecf1308d269531afe5db2d654cc9add926d26cf39dde25b3c1f99c8e0714 I0319 18:02:43.258118 1 recorder.go:75] Recording events/openshift-ingress with fingerprint=2b0f9c9ecaf26f798d3973df9b93721ec058c8604bcc46006bc4305e2941774a I0319 18:02:43.258130 1 recorder.go:75] Recording events/openshift-ingress-canary with fingerprint=b8a5a1de37b6423187b7e95ee9e55ffbb8d4026fb747ce290964a96b75fa954d I0319 18:02:43.258141 1 recorder.go:75] Recording events/openshift-kube-storage-version-migrator with fingerprint=01c23ffa058cd12d0ffe5a787cb93520125c96cc3e697e41b83b0db0de097431 I0319 18:02:43.258188 1 recorder.go:75] Recording events/openshift-kube-storage-version-migrator-operator with fingerprint=2158b348aa343c9749a6e60fd3147e098068e67448856f8deb096268539cf2a4 I0319 18:02:43.258203 1 recorder.go:75] Recording config/pod/openshift-kube-storage-version-migrator/logs/migrator-7d5f866c57-n46dq/migrator_current.log with fingerprint=afe55d0e9a2441f1d4e9af66586d65fe824b4153cc2996020b258e934f10819c I0319 18:02:43.258208 1 recorder.go:75] Recording config/pod/openshift-kube-storage-version-migrator/logs/migrator-7d5f866c57-n46dq/graceful-termination_current.log with fingerprint=9dbe31e3e04fe5007346f60bae831dc04b71dcc08f51d74ff4f6f95d0339831b I0319 18:02:43.258299 1 recorder.go:75] Recording config/pod/openshift-kube-storage-version-migrator-operator/logs/kube-storage-version-migrator-operator-74848b4cb9-8cw7f/kube-storage-version-migrator-operator_current.log with fingerprint=2de79eb53758f5b9b6104df3f2dee4d6be53786cee4e50b0b5c05d6897e0adb1 I0319 18:02:43.258310 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 4.048743704s to process 10 records W0319 18:02:44.225921 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0319 18:02:44.225949 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0319 18:02:44.225970 1 tasks_processing.go:74] worker 13 stopped. E0319 18:02:44.226008 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0319 18:02:44.226034 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 W0319 18:02:44.226052 1 gather.go:155] issue recording gatherer "clusterconfig" function "dvo_metrics" result "config/dvo_metrics" because of the warning: warning: the record with the same fingerprint "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" was already recorded at path "config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt", recording another one with a different path "config/dvo_metrics" I0319 18:02:44.226070 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.019216439s to process 1 records I0319 18:02:51.913742 1 tasks_processing.go:74] worker 19 stopped. I0319 18:02:51.913789 1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b I0319 18:02:51.913806 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.703384341s to process 1 records I0319 18:02:52.094891 1 configmapobserver.go:84] configmaps "insights-config" not found I0319 18:02:52.605854 1 tasks_processing.go:74] worker 30 stopped. I0319 18:02:52.606110 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=e03550ff5cb13d4cd718a41a9755571c983753e7ec8a789b8fb921884bbb0f64 I0319 18:02:52.606127 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.411726685s to process 1 records E0319 18:02:52.606195 1 periodic.go:247] "Unhandled Error" err="clusterconfig failed after 13.425s with: function \"machines\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"support_secret\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0319 18:02:52.607272 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "machines" failed with an error, function "machine_healthchecks" failed with an error, function "support_secret" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0319 18:02:52.607283 1 periodic.go:209] Running workloads gatherer I0319 18:02:52.607298 1 tasks_processing.go:45] number of workers: 2 I0319 18:02:52.607303 1 tasks_processing.go:69] worker 1 listening for tasks. I0319 18:02:52.607307 1 tasks_processing.go:71] worker 1 working on helmchart_info task. I0319 18:02:52.607316 1 tasks_processing.go:69] worker 0 listening for tasks. I0319 18:02:52.607393 1 tasks_processing.go:71] worker 0 working on workload_info task. I0319 18:02:52.640561 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 19s for image data I0319 18:02:52.650560 1 gather_workloads_info.go:387] No image sha256:5a95c19d82767e0235b4edb4a0536482c816904897aae1dc3eb255cb52b87a9f (10ms) I0319 18:02:52.657566 1 tasks_processing.go:74] worker 1 stopped. I0319 18:02:52.657584 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 50.246338ms to process 0 records I0319 18:02:52.660191 1 gather_workloads_info.go:387] No image sha256:a0105d1eb62cf6ac9e5e2ef28d3e89bf6dc514bc594fc7090fe5a5ee18a09c87 (10ms) I0319 18:02:52.669722 1 gather_workloads_info.go:387] No image sha256:765f0d23b637f685f98a31bd47c131b03cf72a40761a3f9a9d6320faa3c33733 (10ms) I0319 18:02:52.682985 1 gather_workloads_info.go:387] No image sha256:2904a78e2eb73fd6a9bb94c105c2a056831fb4113fbb7b0607c50adc9d879c9b (13ms) I0319 18:02:52.692570 1 gather_workloads_info.go:387] No image sha256:a498046d64605bcccee2440aa4f04a4602baaae263cf01d977ec5208e876b1fd (10ms) I0319 18:02:52.705141 1 gather_workloads_info.go:387] No image sha256:1a2532940843248c57d52141185dd71fbc393ab28b65d48f682038632c1dbbad (13ms) I0319 18:02:52.715060 1 gather_workloads_info.go:387] No image sha256:2598489729a4b258e4ecda4a06f6875133f2a10ced5c5241f8a57a8a05418e36 (10ms) I0319 18:02:52.725376 1 gather_workloads_info.go:387] No image sha256:b3909bf664c77097f75b3768830863d642eed3815dab2bfb4415c771ca2d5007 (10ms) I0319 18:02:52.735553 1 gather_workloads_info.go:387] No image sha256:56a85660a445eced5c79a595a0eccf590087c5672d50f49d4c25ad52f9a44f04 (10ms) I0319 18:02:52.746557 1 gather_workloads_info.go:387] No image sha256:e84cb128d930bd1ab867cc89b7b7bf2b2c0e41105ab93b5381069945b3ee9c57 (11ms) I0319 18:02:52.758714 1 gather_workloads_info.go:387] No image sha256:289816958633a763a72dbc44e1dad40466223164e7e253039514f0d974ea5d21 (12ms) I0319 18:02:52.851404 1 gather_workloads_info.go:387] No image sha256:7b31223098f08328f5ddea8e5b871dbbd5f5a61ec550e8956f66793c0c6031a9 (93ms) I0319 18:02:52.951520 1 gather_workloads_info.go:387] No image sha256:943018739e3db1763c3184b460dbc409e058abbac76d57b9927faad317be85e4 (100ms) I0319 18:02:53.051675 1 gather_workloads_info.go:387] No image sha256:653c666f842c13e0baae2e89a9b1efe0e2ef56f621ffb5b32005115d2a26ab8c (100ms) I0319 18:02:53.152329 1 gather_workloads_info.go:387] No image sha256:91828234f107c068c8a4966d08370ae7b73e637651dbc6d92c18c4553402c22c (101ms) I0319 18:02:53.252084 1 gather_workloads_info.go:387] No image sha256:0a99240166165eb5718e7516a43282fe32df9c7c5e809b31b58abe44e42ff94d (100ms) I0319 18:02:53.353105 1 gather_workloads_info.go:387] No image sha256:2e564f336c77116053f34d4201d364d8da04e789cfffa0ea422574c95f2d6404 (101ms) I0319 18:02:53.366365 1 configmapobserver.go:84] configmaps "insights-config" not found I0319 18:02:53.453992 1 gather_workloads_info.go:387] No image sha256:2e57e192c3c1240fd935dcd55c8fde5e70e78bf81d6176c96edf21fafe59f8ba (101ms) I0319 18:02:53.552015 1 gather_workloads_info.go:387] No image sha256:7adc1eab05d6724c76ba751f6df816b08d6e70b78dee9eb94fa6fd9690542c98 (98ms) I0319 18:02:53.564742 1 configmapobserver.go:84] configmaps "insights-config" not found I0319 18:02:53.651749 1 gather_workloads_info.go:387] No image sha256:a258c226562adb14e3a163a1940938526ee6a0928982a7667d85d9a7334ce639 (100ms) I0319 18:02:53.757635 1 gather_workloads_info.go:387] No image sha256:a56211d075aa43cbb491f669a5b2e46ee023dc95b7d51dbac28f463948c5ad61 (106ms) I0319 18:02:53.851167 1 gather_workloads_info.go:387] No image sha256:5f0b67cfbbc381243fb91ccc17345b56d05f4d717c667e8c644e5bf05633ba71 (94ms) I0319 18:02:53.951384 1 gather_workloads_info.go:387] No image sha256:695cf2f0cc07683c2a3ce1eaf3e56fe18abc6e2bac716f7d9843f5d173b9df52 (100ms) I0319 18:02:54.051023 1 gather_workloads_info.go:387] No image sha256:47154813651033d59751fb655a384dbffb64dd26f10bd7f3be0c3128d0486356 (100ms) I0319 18:02:54.151852 1 gather_workloads_info.go:387] No image sha256:4556896f77307821531ef91b7b7faccb82b824ea695693b2989f597f0deca038 (101ms) I0319 18:02:54.252616 1 gather_workloads_info.go:387] No image sha256:36b9e89c3cfcf1ab9ae500486e38afb6862cba48cb0b4d84a09508ab8f3d299f (101ms) I0319 18:02:54.351788 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (99ms) I0319 18:02:54.453991 1 gather_workloads_info.go:387] No image sha256:ca1344cb64140188b7cae7bbc51fb751566c0b0c97d5e39b5850e628032c4a5e (102ms) I0319 18:02:54.555747 1 gather_workloads_info.go:387] No image sha256:521712486e2c6e3c020dad6a1cb340db8e55665b69f7c208fab9cd9e965fd588 (102ms) I0319 18:02:54.651981 1 gather_workloads_info.go:387] No image sha256:03cf4cd7ef1518610c6c7b3ad27d1622d82e98e3dc6e3f8e5d0fceb5c8d3786e (96ms) I0319 18:02:54.753334 1 gather_workloads_info.go:387] No image sha256:c940ea87e7d133d75ba0002ef00c0806825eed3db8094cdb260d1bac18127733 (101ms) I0319 18:02:54.753375 1 tasks_processing.go:74] worker 0 stopped. E0319 18:02:54.753389 1 gather.go:140] gatherer "workloads" function "workload_info" failed with the error: no running pods found for the insights-runtime-extractor statefulset I0319 18:02:54.753865 1 recorder.go:75] Recording config/workload_info with fingerprint=3a0faf2e922ffb9d54ecac17abf5dadea19045dfa754efe705ef1f548fafd236 I0319 18:02:54.753894 1 gather.go:177] gatherer "workloads" function "workload_info" took 2.145963888s to process 1 records E0319 18:02:54.753931 1 periodic.go:247] "Unhandled Error" err="workloads failed after 2.146s with: function \"workload_info\" failed with an error" I0319 18:02:54.755045 1 controllerstatus.go:89] name=periodic-workloads healthy=false reason=PeriodicGatherFailed message=Source workloads could not be retrieved: function "workload_info" failed with an error I0319 18:02:54.755058 1 periodic.go:209] Running conditional gatherer I0319 18:02:54.764115 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules I0319 18:02:54.771354 1 conditional_gatherer.go:107] Get "https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules": dial tcp: lookup console.redhat.com on 172.30.0.10:53: read udp 10.128.0.18:51865->172.30.0.10:53: read: connection refused E0319 18:02:54.771602 1 conditional_gatherer.go:322] unable to update alerts cache: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0319 18:02:54.771662 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0319 18:02:54.779199 1 conditional_gatherer.go:392] cluster version is '4.20.8' E0319 18:02:54.779215 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0319 18:02:54.779223 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0319 18:02:54.779227 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0319 18:02:54.779231 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0319 18:02:54.779253 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0319 18:02:54.779258 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0319 18:02:54.779262 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0319 18:02:54.779266 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0319 18:02:54.779270 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0319 18:02:54.779290 1 tasks_processing.go:45] number of workers: 3 I0319 18:02:54.779313 1 tasks_processing.go:69] worker 2 listening for tasks. I0319 18:02:54.779321 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0319 18:02:54.779331 1 tasks_processing.go:69] worker 0 listening for tasks. I0319 18:02:54.779336 1 tasks_processing.go:69] worker 1 listening for tasks. I0319 18:02:54.779346 1 tasks_processing.go:74] worker 1 stopped. I0319 18:02:54.779346 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0319 18:02:54.779351 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0319 18:02:54.779485 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=7034af97e7e41c22e4b775abdd4b9066c8ebb19da33eb7f69f39bfd2eb5f6406 I0319 18:02:54.779506 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.569µs to process 1 records I0319 18:02:54.779557 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=0394430c431eec4d48bb1811a90918e95161d2282c59af26f2473613cc0959db I0319 18:02:54.779570 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.749µs to process 1 records I0319 18:02:54.779579 1 tasks_processing.go:74] worker 0 stopped. I0319 18:02:54.779613 1 tasks_processing.go:74] worker 2 stopped. I0319 18:02:54.779625 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 258.215µs to process 0 records I0319 18:02:54.779646 1 controllerstatus.go:89] name=periodic-conditional healthy=false reason=NotAvailable message=Get "https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules": dial tcp: lookup console.redhat.com on 172.30.0.10:53: read udp 10.128.0.18:51865->172.30.0.10:53: read: connection refused I0319 18:02:54.779669 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=359de9c990c741675cec72fda96b5c3682221efdb4799f5eaa6e9805bcd3b5c1 I0319 18:02:54.805132 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=0e498702df4c8d820e71271011d8079e13cabfd8dcf68fc920f21fef956cf590 I0319 18:02:54.805312 1 diskrecorder.go:70] Writing 102 records to /var/lib/insights-operator/insights-2026-03-19-180254.tar.gz I0319 18:02:54.812305 1 diskrecorder.go:51] Wrote 102 records to disk in 7ms I0319 18:02:54.812342 1 periodic.go:278] Gathering cluster info every 2h0m0s I0319 18:02:54.812359 1 periodic.go:279] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0319 18:03:07.570987 1 configmapobserver.go:84] configmaps "insights-config" not found I0319 18:03:58.164113 1 observer_polling.go:111] Observed file "/var/run/configmaps/service-ca-bundle/service-ca.crt" has been created (hash="b17497e65131483351c6b0987eaa5a7679c8669b4877c4e5170f4080b3b4d511") W0319 18:03:58.164142 1 builder.go:160] Restart triggered because of file /var/run/configmaps/service-ca-bundle/service-ca.crt was created I0319 18:03:58.164210 1 observer_polling.go:111] Observed file "/var/run/secrets/serving-cert/tls.crt" has been created (hash="6effda0204e077b4ccc77d6867ccc574bc29e7361c3b6d9933724d1ab05ee6e5") I0319 18:03:58.164224 1 genericapiserver.go:548] "[graceful-termination] shutdown event" name="ShutdownInitiated"