I0331 04:03:07.502394 1 cmd.go:253] Using service-serving-cert provided certificates I0331 04:03:07.502821 1 observer_polling.go:159] Starting file observer I0331 04:03:07.835655 1 operator.go:60] Starting insights-operator v0.0.0-master+$Format:%H$ I0331 04:03:07.835861 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0331 04:03:07.836391 1 secure_serving.go:57] Forcing use of http/1.1 only W0331 04:03:07.836412 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0331 04:03:07.836416 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0331 04:03:07.836420 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0331 04:03:07.836422 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0331 04:03:07.836425 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0331 04:03:07.836427 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0331 04:03:07.836441 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0331 04:03:07.839881 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"98f53846-1ae3-4825-b80a-8161bb1de2f5", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}} I0331 04:03:07.839894 1 operator.go:125] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GatewayAPI GatewayAPIController HighlyAvailableArbiter ImageVolume IngressControllerLBSubnetsAWS KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController SigstoreImageVerification StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPClusterHostedDNSInstall GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall ImageModeStatusReporting ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesAzure ManagedBootImagesvSphere MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutatingAdmissionPolicy NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterOperations NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PreconfiguredUDNAddresses SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeAttributesClass VolumeGroupSnapshot] I0331 04:03:07.841695 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0331 04:03:07.841672 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0331 04:03:07.841723 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0331 04:03:07.841697 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0331 04:03:07.841719 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0331 04:03:07.842128 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0331 04:03:07.842328 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0331 04:03:07.842578 1 secure_serving.go:213] Serving securely on [::]:8443 I0331 04:03:07.842609 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0331 04:03:07.846464 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0331 04:03:07.846485 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0331 04:03:07.846576 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0331 04:03:07.852709 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0331 04:03:07.852734 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0331 04:03:07.856322 1 secretconfigobserver.go:119] support secret does not exist I0331 04:03:07.859961 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0331 04:03:07.863468 1 secretconfigobserver.go:119] support secret does not exist I0331 04:03:07.865680 1 recorder.go:161] Pruning old reports every 6h45m16s, max age is 288h0m0s I0331 04:03:07.869861 1 periodic.go:209] Running clusterconfig gatherer I0331 04:03:07.869906 1 tasks_processing.go:45] number of workers: 64 I0331 04:03:07.869948 1 tasks_processing.go:69] worker 20 listening for tasks. I0331 04:03:07.869958 1 tasks_processing.go:71] worker 20 working on openshift_machine_api_events task. I0331 04:03:07.869964 1 tasks_processing.go:69] worker 42 listening for tasks. I0331 04:03:07.869971 1 tasks_processing.go:69] worker 41 listening for tasks. I0331 04:03:07.869977 1 tasks_processing.go:69] worker 44 listening for tasks. I0331 04:03:07.869973 1 tasks_processing.go:69] worker 43 listening for tasks. I0331 04:03:07.869859 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0331 04:03:07.869983 1 tasks_processing.go:69] worker 9 listening for tasks. I0331 04:03:07.869859 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0331 04:03:07.869990 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0331 04:03:07.869997 1 insightsreport.go:296] Starting report retriever I0331 04:03:07.869999 1 tasks_processing.go:69] worker 24 listening for tasks. I0331 04:03:07.870002 1 tasks_processing.go:69] worker 53 listening for tasks. I0331 04:03:07.870009 1 tasks_processing.go:69] worker 54 listening for tasks. I0331 04:03:07.870006 1 tasks_processing.go:69] worker 25 listening for tasks. I0331 04:03:07.870015 1 tasks_processing.go:69] worker 27 listening for tasks. I0331 04:03:07.869989 1 tasks_processing.go:69] worker 22 listening for tasks. I0331 04:03:07.870020 1 tasks_processing.go:69] worker 55 listening for tasks. I0331 04:03:07.870019 1 tasks_processing.go:69] worker 45 listening for tasks. I0331 04:03:07.870025 1 tasks_processing.go:69] worker 1 listening for tasks. I0331 04:03:07.870026 1 tasks_processing.go:69] worker 50 listening for tasks. I0331 04:03:07.870033 1 tasks_processing.go:69] worker 46 listening for tasks. I0331 04:03:07.870036 1 tasks_processing.go:69] worker 2 listening for tasks. I0331 04:03:07.870035 1 tasks_processing.go:69] worker 52 listening for tasks. I0331 04:03:07.870039 1 tasks_processing.go:69] worker 49 listening for tasks. I0331 04:03:07.870042 1 tasks_processing.go:69] worker 38 listening for tasks. I0331 04:03:07.870006 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0331 04:03:07.870046 1 tasks_processing.go:69] worker 58 listening for tasks. I0331 04:03:07.870049 1 tasks_processing.go:69] worker 15 listening for tasks. I0331 04:03:07.870053 1 tasks_processing.go:69] worker 39 listening for tasks. I0331 04:03:07.870038 1 tasks_processing.go:69] worker 37 listening for tasks. I0331 04:03:07.870056 1 tasks_processing.go:69] worker 8 listening for tasks. I0331 04:03:07.870028 1 tasks_processing.go:69] worker 35 listening for tasks. I0331 04:03:07.870010 1 tasks_processing.go:69] worker 26 listening for tasks. I0331 04:03:07.870010 1 tasks_processing.go:69] worker 48 listening for tasks. I0331 04:03:07.870017 1 tasks_processing.go:69] worker 0 listening for tasks. I0331 04:03:07.870024 1 tasks_processing.go:69] worker 28 listening for tasks. I0331 04:03:07.870027 1 tasks_processing.go:69] worker 51 listening for tasks. I0331 04:03:07.870029 1 tasks_processing.go:69] worker 56 listening for tasks. I0331 04:03:07.870033 1 tasks_processing.go:69] worker 36 listening for tasks. I0331 04:03:07.870038 1 tasks_processing.go:69] worker 57 listening for tasks. I0331 04:03:07.870045 1 tasks_processing.go:69] worker 6 listening for tasks. I0331 04:03:07.870079 1 tasks_processing.go:69] worker 3 listening for tasks. I0331 04:03:07.870045 1 tasks_processing.go:69] worker 34 listening for tasks. I0331 04:03:07.870056 1 tasks_processing.go:69] worker 59 listening for tasks. I0331 04:03:07.870091 1 tasks_processing.go:69] worker 12 listening for tasks. I0331 04:03:07.870060 1 tasks_processing.go:69] worker 29 listening for tasks. I0331 04:03:07.870065 1 tasks_processing.go:69] worker 60 listening for tasks. I0331 04:03:07.870065 1 tasks_processing.go:69] worker 10 listening for tasks. I0331 04:03:07.870067 1 tasks_processing.go:69] worker 30 listening for tasks. I0331 04:03:07.870097 1 tasks_processing.go:69] worker 32 listening for tasks. I0331 04:03:07.870108 1 tasks_processing.go:69] worker 62 listening for tasks. I0331 04:03:07.870085 1 tasks_processing.go:69] worker 13 listening for tasks. I0331 04:03:07.869984 1 tasks_processing.go:69] worker 21 listening for tasks. I0331 04:03:07.870116 1 tasks_processing.go:69] worker 14 listening for tasks. I0331 04:03:07.869994 1 tasks_processing.go:69] worker 23 listening for tasks. I0331 04:03:07.870127 1 tasks_processing.go:69] worker 17 listening for tasks. I0331 04:03:07.870135 1 tasks_processing.go:69] worker 47 listening for tasks. I0331 04:03:07.870142 1 tasks_processing.go:71] worker 24 working on openstack_version task. I0331 04:03:07.870151 1 tasks_processing.go:69] worker 16 listening for tasks. I0331 04:03:07.870158 1 tasks_processing.go:69] worker 19 listening for tasks. I0331 04:03:07.870161 1 tasks_processing.go:71] worker 41 working on cost_management_metrics_configs task. I0331 04:03:07.870169 1 tasks_processing.go:71] worker 43 working on olm_operators task. I0331 04:03:07.870176 1 tasks_processing.go:71] worker 9 working on nodes task. I0331 04:03:07.870184 1 tasks_processing.go:69] worker 33 listening for tasks. I0331 04:03:07.870193 1 tasks_processing.go:71] worker 33 working on cluster_apiserver task. I0331 04:03:07.869959 1 tasks_processing.go:69] worker 63 listening for tasks. I0331 04:03:07.870168 1 tasks_processing.go:71] worker 19 working on version task. I0331 04:03:07.870211 1 tasks_processing.go:69] worker 61 listening for tasks. I0331 04:03:07.870218 1 tasks_processing.go:71] worker 25 working on sap_pods task. I0331 04:03:07.870220 1 tasks_processing.go:69] worker 18 listening for tasks. I0331 04:03:07.870226 1 tasks_processing.go:71] worker 29 working on oauths task. I0331 04:03:07.870224 1 tasks_processing.go:69] worker 11 listening for tasks. I0331 04:03:07.870149 1 tasks_processing.go:71] worker 44 working on container_images task. I0331 04:03:07.870242 1 tasks_processing.go:71] worker 58 working on lokistack task. I0331 04:03:07.870498 1 tasks_processing.go:69] worker 5 listening for tasks. I0331 04:03:07.870422 1 tasks_processing.go:69] worker 7 listening for tasks. I0331 04:03:07.871151 1 tasks_processing.go:71] worker 52 working on validating_webhook_configurations task. I0331 04:03:07.871158 1 tasks_processing.go:71] worker 39 working on install_plans task. I0331 04:03:07.870225 1 tasks_processing.go:71] worker 28 working on storage_classes task. I0331 04:03:07.870625 1 tasks_processing.go:71] worker 63 working on qemu_kubevirt_launcher_logs task. I0331 04:03:07.870209 1 tasks_processing.go:69] worker 4 listening for tasks. I0331 04:03:07.871548 1 tasks_processing.go:71] worker 4 working on openshift_logging task. I0331 04:03:07.870158 1 tasks_processing.go:71] worker 42 working on pod_network_connectivity_checks task. I0331 04:03:07.870520 1 tasks_processing.go:71] worker 46 working on monitoring_persistent_volumes task. I0331 04:03:07.870548 1 tasks_processing.go:71] worker 5 working on networks task. I0331 04:03:07.870242 1 tasks_processing.go:71] worker 15 working on ingress task. I0331 04:03:07.870211 1 tasks_processing.go:71] worker 53 working on sap_datahubs task. I0331 04:03:07.870215 1 tasks_processing.go:71] worker 54 working on schedulers task. I0331 04:03:07.870162 1 tasks_processing.go:71] worker 16 working on image task. I0331 04:03:07.870229 1 tasks_processing.go:71] worker 27 working on overlapping_namespace_uids task. I0331 04:03:07.870232 1 tasks_processing.go:71] worker 22 working on openstack_dataplanedeployments task. I0331 04:03:07.870237 1 tasks_processing.go:71] worker 55 working on operators task. I0331 04:03:07.870239 1 tasks_processing.go:71] worker 51 working on support_secret task. I0331 04:03:07.870340 1 tasks_processing.go:71] worker 62 working on sap_config task. I0331 04:03:07.870457 1 tasks_processing.go:71] worker 36 working on infrastructures task. I0331 04:03:07.870548 1 tasks_processing.go:71] worker 57 working on feature_gates task. I0331 04:03:07.870590 1 tasks_processing.go:71] worker 11 working on ingress_certificates task. I0331 04:03:07.870434 1 tasks_processing.go:69] worker 40 listening for tasks. I0331 04:03:07.872923 1 tasks_processing.go:71] worker 40 working on openstack_controlplanes task. I0331 04:03:07.870635 1 tasks_processing.go:71] worker 37 working on machines task. I0331 04:03:07.870705 1 tasks_processing.go:71] worker 2 working on container_runtime_configs task. I0331 04:03:07.870732 1 tasks_processing.go:71] worker 8 working on machine_config_pools task. I0331 04:03:07.870742 1 tasks_processing.go:71] worker 34 working on service_accounts task. I0331 04:03:07.870744 1 tasks_processing.go:71] worker 45 working on clusterroles task. I0331 04:03:07.870747 1 tasks_processing.go:71] worker 3 working on proxies task. I0331 04:03:07.870777 1 tasks_processing.go:71] worker 35 working on aggregated_monitoring_cr_names task. I0331 04:03:07.870796 1 tasks_processing.go:71] worker 6 working on nodenetworkconfigurationpolicies task. I0331 04:03:07.870801 1 tasks_processing.go:71] worker 1 working on silenced_alerts task. I0331 04:03:07.870862 1 tasks_processing.go:71] worker 23 working on jaegers task. I0331 04:03:07.870861 1 tasks_processing.go:71] worker 18 working on openstack_dataplanenodesets task. I0331 04:03:07.870868 1 tasks_processing.go:71] worker 61 working on metrics task. I0331 04:03:07.870871 1 tasks_processing.go:71] worker 50 working on machine_configs task. I0331 04:03:07.870869 1 tasks_processing.go:71] worker 13 working on certificate_signing_requests task. I0331 04:03:07.870932 1 tasks_processing.go:71] worker 17 working on storage_cluster task. I0331 04:03:07.870927 1 tasks_processing.go:71] worker 21 working on image_registries task. I0331 04:03:07.870944 1 tasks_processing.go:71] worker 10 working on dvo_metrics task. I0331 04:03:07.870958 1 tasks_processing.go:71] worker 14 working on operators_pods_and_events task. I0331 04:03:07.870201 1 tasks_processing.go:71] worker 47 working on mutating_webhook_configurations task. I0331 04:03:07.870968 1 tasks_processing.go:71] worker 49 working on config_maps task. I0331 04:03:07.870998 1 tasks_processing.go:71] worker 38 working on node_logs task. I0331 04:03:07.871001 1 tasks_processing.go:71] worker 60 working on authentication task. I0331 04:03:07.874441 1 tasks_processing.go:71] worker 24 working on machine_healthchecks task. I0331 04:03:07.874486 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 4.290537ms to process 0 records I0331 04:03:07.874499 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 4.006229ms to process 0 records I0331 04:03:07.871009 1 tasks_processing.go:71] worker 30 working on machine_sets task. I0331 04:03:07.874549 1 tasks_processing.go:71] worker 58 working on crds task. I0331 04:03:07.871006 1 tasks_processing.go:71] worker 12 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0331 04:03:07.871035 1 tasks_processing.go:71] worker 59 working on ceph_cluster task. I0331 04:03:07.871046 1 tasks_processing.go:71] worker 26 working on active_alerts task. I0331 04:03:07.871084 1 tasks_processing.go:71] worker 48 working on tsdb_status task. I0331 04:03:07.871084 1 tasks_processing.go:71] worker 32 working on machine_autoscalers task. I0331 04:03:07.871095 1 tasks_processing.go:71] worker 0 working on pdbs task. I0331 04:03:07.871155 1 tasks_processing.go:71] worker 7 working on image_pruners task. I0331 04:03:07.870239 1 tasks_processing.go:69] worker 31 listening for tasks. I0331 04:03:07.874990 1 tasks_processing.go:74] worker 31 stopped. I0331 04:03:07.870243 1 tasks_processing.go:71] worker 56 working on nodenetworkstates task. I0331 04:03:07.878892 1 tasks_processing.go:74] worker 4 stopped. I0331 04:03:07.878906 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 7.335798ms to process 0 records I0331 04:03:07.878914 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 8.732426ms to process 0 records I0331 04:03:07.878919 1 tasks_processing.go:74] worker 41 stopped. I0331 04:03:07.878948 1 tasks_processing.go:74] worker 53 stopped. I0331 04:03:07.878968 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 6.828039ms to process 0 records I0331 04:03:07.878995 1 tasks_processing.go:74] worker 42 stopped. E0331 04:03:07.879110 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0331 04:03:07.879126 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 7.359066ms to process 0 records I0331 04:03:07.879135 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 9.075378ms to process 0 records I0331 04:03:07.879143 1 tasks_processing.go:74] worker 20 stopped. I0331 04:03:07.879212 1 controller.go:128] Initializing last reported time to 0001-01-01T00:00:00Z I0331 04:03:07.879229 1 controller.go:326] The initial operator extension status is healthy I0331 04:03:07.879239 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0331 04:03:07.879248 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0331 04:03:07.879253 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0331 04:03:07.879265 1 tasks_processing.go:74] worker 29 stopped. I0331 04:03:07.879275 1 controller.go:489] The operator is still being initialized I0331 04:03:07.879283 1 controller.go:512] The operator is healthy I0331 04:03:07.879341 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0331 04:03:07.879344 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0331 04:03:07.879533 1 operator.go:288] started I0331 04:03:07.879557 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0331 04:03:07.879607 1 recorder.go:75] Recording config/oauth with fingerprint=ef2136d7a6cee525ccf3e8279a7d97ed1606b4c5721251ff4f000ad161ffba73 I0331 04:03:07.879625 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 9.020685ms to process 1 records I0331 04:03:07.881305 1 gather_logs.go:145] no pods in namespace were found I0331 04:03:07.881319 1 tasks_processing.go:74] worker 63 stopped. I0331 04:03:07.881328 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 9.868146ms to process 0 records I0331 04:03:07.881473 1 tasks_processing.go:74] worker 43 stopped. I0331 04:03:07.881485 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 11.28984ms to process 0 records I0331 04:03:07.881570 1 tasks_processing.go:74] worker 33 stopped. I0331 04:03:07.881689 1 recorder.go:75] Recording config/apiserver with fingerprint=86b266957cb7ed238db3c30b75d4c18ac0a95c2da1a68db7bb07692741d12962 I0331 04:03:07.881700 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 11.369909ms to process 1 records I0331 04:03:07.881710 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 11.389886ms to process 0 records I0331 04:03:07.881715 1 tasks_processing.go:74] worker 25 stopped. E0331 04:03:07.881757 1 gather_active_alerts.go:64] Unable to retrieve most recent alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp: lookup alertmanager-main.openshift-monitoring.svc on 172.30.0.10:53: no such host I0331 04:03:07.881775 1 tasks_processing.go:74] worker 26 stopped. E0331 04:03:07.881778 1 gather_silenced_alerts.go:52] Unable to retrieve silenced alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp: lookup alertmanager-main.openshift-monitoring.svc on 172.30.0.10:53: no such host E0331 04:03:07.881781 1 gather.go:140] gatherer "clusterconfig" function "active_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp: lookup alertmanager-main.openshift-monitoring.svc on 172.30.0.10:53: no such host I0331 04:03:07.881801 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 7.063432ms to process 0 records E0331 04:03:07.881831 1 gather.go:140] gatherer "clusterconfig" function "silenced_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp: lookup alertmanager-main.openshift-monitoring.svc on 172.30.0.10:53: no such host I0331 04:03:07.881840 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 8.200321ms to process 0 records I0331 04:03:07.881848 1 tasks_processing.go:74] worker 1 stopped. I0331 04:03:07.898435 1 tasks_processing.go:74] worker 40 stopped. I0331 04:03:07.898444 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 25.504535ms to process 0 records I0331 04:03:07.898505 1 tasks_processing.go:74] worker 62 stopped. I0331 04:03:07.898519 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 25.888456ms to process 0 records I0331 04:03:07.898603 1 tasks_processing.go:74] worker 2 stopped. I0331 04:03:07.899000 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 25.220204ms to process 0 records E0331 04:03:07.899022 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0331 04:03:07.899031 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 26.077596ms to process 0 records I0331 04:03:07.899046 1 tasks_processing.go:74] worker 17 stopped. I0331 04:03:07.899076 1 tasks_processing.go:74] worker 51 stopped. I0331 04:03:07.899139 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 24.84565ms to process 0 records I0331 04:03:07.899169 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 25.166712ms to process 0 records I0331 04:03:07.899216 1 tasks_processing.go:74] worker 18 stopped. I0331 04:03:07.899322 1 tasks_processing.go:74] worker 36 stopped. I0331 04:03:07.900527 1 recorder.go:75] Recording config/infrastructure with fingerprint=687bfe99ab765714993b25d3aa881a606d870735c8dc68531b9aa5a0fe26af9f I0331 04:03:07.900553 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 26.317093ms to process 1 records I0331 04:03:07.900564 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 25.302123ms to process 0 records E0331 04:03:07.900582 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0331 04:03:07.900591 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 24.622346ms to process 0 records I0331 04:03:07.900602 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 24.571356ms to process 0 records E0331 04:03:07.900609 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0331 04:03:07.900734 1 tasks_processing.go:74] worker 37 stopped. I0331 04:03:07.900739 1 gather.go:177] gatherer "clusterconfig" function "machines" took 26.07058ms to process 0 records I0331 04:03:07.900710 1 tasks_processing.go:74] worker 23 stopped. I0331 04:03:07.900740 1 tasks_processing.go:74] worker 30 stopped. I0331 04:03:07.900698 1 tasks_processing.go:74] worker 24 stopped. I0331 04:03:07.901097 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=9f8f4dc8999bd703821dd94eb205e5c5795380d00d375d08cbf4617a13145c05 I0331 04:03:07.901142 1 tasks_processing.go:74] worker 54 stopped. I0331 04:03:07.901290 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 27.222309ms to process 1 records I0331 04:03:07.901473 1 recorder.go:75] Recording config/image with fingerprint=8a8205f36c7e6e4d6bf29f2629023ad2e74b3f6f28f149b8ac209a016e9ea94c I0331 04:03:07.901492 1 gather.go:177] gatherer "clusterconfig" function "image" took 27.165174ms to process 1 records I0331 04:03:07.901482 1 tasks_processing.go:74] worker 16 stopped. I0331 04:03:07.901513 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 24.691402ms to process 0 records I0331 04:03:07.901540 1 tasks_processing.go:74] worker 22 stopped. I0331 04:03:07.901543 1 tasks_processing.go:74] worker 32 stopped. I0331 04:03:07.901608 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 27.234597ms to process 0 records I0331 04:03:07.901625 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 27.74151ms to process 0 records I0331 04:03:07.901634 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 24.934036ms to process 0 records I0331 04:03:07.901643 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 24.695748ms to process 0 records I0331 04:03:07.901644 1 tasks_processing.go:74] worker 46 stopped. I0331 04:03:07.901658 1 tasks_processing.go:74] worker 59 stopped. I0331 04:03:07.901664 1 tasks_processing.go:74] worker 56 stopped. I0331 04:03:07.901731 1 tasks_processing.go:74] worker 28 stopped. I0331 04:03:07.902291 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=dbcd15edeb74657633614da9f0a1e271a8ab246752e448f8cd92368576a7c721 I0331 04:03:07.902313 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=fbbe976e82c29383aeb6b24554c8c90208aa5520e7cb3680fdcbe62ae30f20d6 I0331 04:03:07.902319 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 28.466192ms to process 2 records I0331 04:03:07.902329 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 26.316538ms to process 0 records E0331 04:03:07.902342 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0331 04:03:07.902353 1 tasks_processing.go:74] worker 6 stopped. E0331 04:03:07.902383 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0331 04:03:07.902424 1 tasks_processing.go:74] worker 15 stopped. I0331 04:03:07.902523 1 recorder.go:75] Recording config/ingress with fingerprint=3d9708e7eeed84e1911c28e8feb0da07eba6de803e7a3d99c1d1b01cfce3178f I0331 04:03:07.902534 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 28.629319ms to process 1 records I0331 04:03:07.902684 1 tasks_processing.go:74] worker 60 stopped. I0331 04:03:07.902723 1 recorder.go:75] Recording config/authentication with fingerprint=86fcd1f4628fd102f2bc54b36384c00cd49da051919c3ebf7e6feb52d3b354a2 I0331 04:03:07.902739 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 26.895717ms to process 1 records I0331 04:03:07.902827 1 tasks_processing.go:74] worker 19 stopped. I0331 04:03:07.903057 1 recorder.go:75] Recording config/version with fingerprint=2f92c0cd78d04e355c85e25af05943d70c5dc71aeb0a619f4712ca2695d35a5f I0331 04:03:07.903107 1 recorder.go:75] Recording config/id with fingerprint=5431ecf793298a8a89e6a7863c38e250dcc556eb7a775599b57aeef48df2511e I0331 04:03:07.903132 1 gather.go:177] gatherer "clusterconfig" function "version" took 31.171685ms to process 2 records I0331 04:03:07.903230 1 tasks_processing.go:74] worker 5 stopped. I0331 04:03:07.903329 1 recorder.go:75] Recording config/network with fingerprint=60397291cc1c8edb79a46d4c1570a30989edc6651fbc683aa9e926d30562fdca I0331 04:03:07.903345 1 gather.go:177] gatherer "clusterconfig" function "networks" took 29.40669ms to process 1 records I0331 04:03:07.903445 1 tasks_processing.go:74] worker 9 stopped. I0331 04:03:07.903973 1 recorder.go:75] Recording config/node/ip-10-0-1-149.ec2.internal with fingerprint=60be2153e057976d98b7d00e49aedc67a173929540483b21fa57bd47838ea666 I0331 04:03:07.903994 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 31.275924ms to process 1 records I0331 04:03:07.904251 1 tasks_processing.go:74] worker 7 stopped. I0331 04:03:07.904289 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=416bb0a4b7049d9521c042ccadfbad9b7525a69f34539608f95e82733b9fac47 I0331 04:03:07.904330 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 26.503011ms to process 1 records I0331 04:03:07.904473 1 tasks_processing.go:74] worker 57 stopped. I0331 04:03:07.904530 1 recorder.go:75] Recording config/featuregate with fingerprint=7af54226de7534aeb8f1a1411863df98b4875ce3c24c9be3d7ad2f59e6129b4d I0331 04:03:07.904554 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 28.896164ms to process 1 records E0331 04:03:07.904564 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0331 04:03:07.904591 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 27.562426ms to process 0 records E0331 04:03:07.904604 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0331 04:03:07.904626 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 28.502747ms to process 0 records I0331 04:03:07.904641 1 tasks_processing.go:74] worker 48 stopped. I0331 04:03:07.904652 1 tasks_processing.go:74] worker 61 stopped. I0331 04:03:07.904829 1 tasks_processing.go:74] worker 52 stopped. I0331 04:03:07.904882 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=315ba5e171ae02a48f4f87c8cce580910619c4b3188e1bd2448c9b7e35ac079d I0331 04:03:07.905003 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=e99a7d40e9b493f20bfcaab57e67558725138992f8198f85d11b8ca66c5af26c W0331 04:03:07.905505 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0331 04:03:07.905652 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=3706484a2ce8c98809184cee6c8ecba046c0e55ea13aa7b55fdadd4390e9ba92 I0331 04:03:07.905732 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=b51b001ddfc64a8ed2bade6c3aebb58bc6f7539126e9a8bdf9548862de4acf0f I0331 04:03:07.905870 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=9ba465cf1f5ecd394d4f45e6a9870858a26204320d895be762c8ba7b48d22d2d I0331 04:03:07.905915 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=c5097be40ad12e3c02db86c817634fa48b9469520d00279ac89c6ed4809f4348 I0331 04:03:07.905962 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=d182d4020ca8df2515811617b9b08193764eea2b35f31b8dd32951e5bf1e97fd I0331 04:03:07.906124 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=7e94d42a31bad7b9439d98bcf5860cc9d6e62f8ddb30ce141c1ef7c68460332a I0331 04:03:07.906182 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=5274ac70e44d582477a11a515df1abc7a7288e041a26cb6714c51e50fbf937da I0331 04:03:07.906236 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=52cdd6906adde926523000747830548271aa4d7077f88663797aa3f0c204b0af I0331 04:03:07.906326 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=17e8fc4ecc05f861b5b642810c37ae39fca6336d62b4f4bf664fdebf44bd0c52 I0331 04:03:07.906386 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 31.69241ms to process 11 records I0331 04:03:07.906425 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 28.556758ms to process 0 records I0331 04:03:07.906634 1 tasks_processing.go:74] worker 38 stopped. I0331 04:03:07.906761 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=d9b204a8b8f30d08b97308472d15f7e58170cfeec9d13b829aa2b4459a4ac21b I0331 04:03:07.906835 1 tasks_processing.go:74] worker 0 stopped. I0331 04:03:07.906846 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=43c64b13cf68de786ddc376b4354b7a96d2bc1dfb9cdac54d57a18d2dc2c783d I0331 04:03:07.906918 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=8ec4771a8551898488683c02d27f3f76e129275f5cd59d9277dea686561b32a5 I0331 04:03:07.907041 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=1395b6e6575df9c21e2db7074d210c9af3f685ffd60e6e28c86c7730b708c7b5 I0331 04:03:07.907100 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=fe133580dd8cb486cc39228721fa9df985fb6677353bd0477d2fcfa1b33ac16f I0331 04:03:07.907150 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=0a620b17f4b60b0d09fc18114258c8a65590bb065a18d503a3b4605c5c91f427 I0331 04:03:07.907187 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 29.112626ms to process 6 records I0331 04:03:07.907385 1 recorder.go:75] Recording config/proxy with fingerprint=eb0a301c83f85e86ac5bf14b6e6bc96ae7cc547d31600025e9a41489bbeebb8e I0331 04:03:07.907420 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 30.585292ms to process 1 records I0331 04:03:07.907599 1 tasks_processing.go:74] worker 3 stopped. I0331 04:03:07.907703 1 tasks_processing.go:74] worker 47 stopped. I0331 04:03:07.907757 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0331 04:03:07.907777 1 controller.go:212] Source scaController *sca.Controller is not ready I0331 04:03:07.907784 1 controller.go:212] Source clusterTransferController *clustertransfer.Controller is not ready I0331 04:03:07.907794 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0331 04:03:07.907801 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0331 04:03:07.907874 1 controller.go:489] The operator is still being initialized I0331 04:03:07.907886 1 controller.go:512] The operator is healthy I0331 04:03:07.907898 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=a7afa48490015e4c484da5294f3310b129cf99c7e3619948c6087e552d757261 I0331 04:03:07.907975 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=b2fba9d8ac47b64ce7bd043f5bfbcb7016b8a7cc428caf11627930d3e04cda83 I0331 04:03:07.908046 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=9d61e4c21e88721557c94ef9b271922cb4ae0e66ea2a5376e9bc368210ab21c0 I0331 04:03:07.908056 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 30.033238ms to process 3 records I0331 04:03:07.908066 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 30.811255ms to process 0 records I0331 04:03:07.908092 1 tasks_processing.go:74] worker 8 stopped. I0331 04:03:07.908155 1 tasks_processing.go:74] worker 21 stopped. I0331 04:03:07.909418 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=ddc92c1b702139b6665eb28a702d66e69eb515cb31a7cbf437528f99f16aa66b I0331 04:03:07.909449 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 30.932242ms to process 1 records I0331 04:03:07.909473 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0331 04:03:07.909490 1 tasks_processing.go:74] worker 27 stopped. I0331 04:03:07.909494 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 33.561487ms to process 1 records I0331 04:03:07.909511 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 34.209857ms to process 0 records I0331 04:03:07.909519 1 tasks_processing.go:74] worker 13 stopped. I0331 04:03:07.913086 1 tasks_processing.go:74] worker 44 stopped. I0331 04:03:07.914478 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-575cd97545-rp4p4 with fingerprint=5a154d7ed31a419786f44bc01cb1c1d95113b8cebbe63b9c905cb10d4df59c42 I0331 04:03:07.914544 1 recorder.go:75] Recording config/running_containers with fingerprint=7fa2256330b189ba09c7a6bd975365a793fef167b81f4b8965ba0cca6cd230f9 I0331 04:03:07.914555 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 42.831631ms to process 2 records I0331 04:03:07.919574 1 tasks_processing.go:74] worker 12 stopped. I0331 04:03:07.919588 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 44.986713ms to process 0 records I0331 04:03:07.926430 1 tasks_processing.go:74] worker 45 stopped. I0331 04:03:07.926717 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=ff9419f7c74cc2eebea47b6df8a5afd13749c9eb5b2a06244460a6536c6b4a89 I0331 04:03:07.926896 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=99b01da62689457d469b164803240da0c2d5d05ecb23b4ac393a0df233912418 I0331 04:03:07.926913 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 52.950267ms to process 2 records I0331 04:03:07.926968 1 tasks_processing.go:74] worker 58 stopped. I0331 04:03:07.928621 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=48d3d8c208abc31980a9a6cfc25296bbe09c20742a4149c79b95f913e0c14d43 I0331 04:03:07.928901 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=540d0f645ecea543201776b3f3582ef332b6ea876bbf733ddbe916206cf46b46 I0331 04:03:07.928914 1 gather.go:177] gatherer "clusterconfig" function "crds" took 52.40786ms to process 2 records I0331 04:03:07.928925 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 53.720467ms to process 0 records I0331 04:03:07.928940 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0331 04:03:07.928945 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 54.338363ms to process 1 records I0331 04:03:07.928942 1 tasks_processing.go:74] worker 35 stopped. I0331 04:03:07.928951 1 tasks_processing.go:74] worker 50 stopped. I0331 04:03:07.932070 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0331 04:03:07.937309 1 tasks_processing.go:74] worker 49 stopped. E0331 04:03:07.937321 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0331 04:03:07.937328 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0331 04:03:07.937332 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0331 04:03:07.937340 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=26b6661162b099a0f5a279859b4f46c867929a79d9a4a41fde4be4e6fe138018 I0331 04:03:07.937364 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0331 04:03:07.937371 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0331 04:03:07.937376 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=c93090eb0d2a4736885abeb79c91680cfd01fda46464f83456b085d4dc8239f0 I0331 04:03:07.937384 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0331 04:03:07.937420 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0331 04:03:07.937428 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0331 04:03:07.937433 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 63.088481ms to process 7 records I0331 04:03:07.942338 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0331 04:03:07.942348 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0331 04:03:07.942359 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0331 04:03:07.943117 1 tasks_processing.go:74] worker 11 stopped. E0331 04:03:07.943130 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0331 04:03:07.943135 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2pcfa8c9vnjo834h2rmsfk75o9078tj0-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2pcfa8c9vnjo834h2rmsfk75o9078tj0-primary-cert-bundle-secret" not found I0331 04:03:07.943178 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=c05930e56683f9d15bdbea28ee8d6ddb95f2bd7275d34dbfb982f1d18a0a9d53 I0331 04:03:07.943190 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 70.275494ms to process 1 records I0331 04:03:07.947004 1 base_controller.go:82] Caches are synced for ConfigController I0331 04:03:07.947015 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0331 04:03:07.951126 1 configmapobserver.go:84] configmaps "insights-config" not found I0331 04:03:07.979582 1 base_controller.go:82] Caches are synced for LoggingSyncer I0331 04:03:07.979594 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0331 04:03:07.990318 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0331 04:03:07.993942 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0331 04:03:08.035717 1 gather_cluster_operator_pods_and_events.go:121] Found 5 pods with 5 containers I0331 04:03:08.035727 1 gather_cluster_operator_pods_and_events.go:235] Maximum buffer size: 5033164 bytes I0331 04:03:08.037356 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-6857fd4c59-ml2j2 pod in namespace openshift-ingress (previous: false). I0331 04:03:08.050912 1 cluster_transfer.go:102] no available accepted cluster transfer I0331 04:03:08.050925 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0331 04:03:08.067754 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-6857fd4c59-rzfkp pod in namespace openshift-ingress (previous: false). I0331 04:03:08.071721 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0331 04:03:08.071733 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-6df66bc69f-fvkf2 pod in namespace openshift-ingress (previous: false). I0331 04:03:08.083003 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-6df66bc69f-vvh9t pod in namespace openshift-ingress (previous: false). I0331 04:03:08.086945 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0331 04:03:08.086958 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-vpnxg pod in namespace openshift-ingress-canary (previous: false). I0331 04:03:08.109012 1 sca.go:266] etc-pki-entitlement secret successfully created I0331 04:03:08.109026 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated I0331 04:03:08.117243 1 tasks_processing.go:74] worker 14 stopped. I0331 04:03:08.117347 1 recorder.go:75] Recording events/openshift-ingress-operator with fingerprint=65f1c18d88b7ebc1113512c29c3e6ce81ffe34be352d82a6e405e3955d1181b4 I0331 04:03:08.117416 1 recorder.go:75] Recording events/openshift-ingress with fingerprint=d9c8d8c04b81bff770eca423018b1516e4c243cd19dc846b330d4f60fb21751d I0331 04:03:08.117436 1 recorder.go:75] Recording events/openshift-ingress-canary with fingerprint=4cbf42a4730666bd5d23109ac25015b43a911a7f6701b7b7aaab8991ba4da8e5 I0331 04:03:08.117458 1 recorder.go:75] Recording config/pod/openshift-ingress/logs/router-default-6857fd4c59-ml2j2/router_current.log with fingerprint=9bbc4df34bc635cfda672b8d9f2c5b9dbc92aebd9b1d57e90f8e12b34353d5ac I0331 04:03:08.117499 1 recorder.go:75] Recording config/pod/openshift-ingress/logs/router-default-6df66bc69f-fvkf2/router_current.log with fingerprint=4c09916fb879e787914085a0f857a35acebe37bb2c532c626716d2e727845154 I0331 04:03:08.117505 1 recorder.go:75] Recording config/pod/openshift-ingress-canary/logs/ingress-canary-vpnxg/serve-healthcheck-canary_current.log with fingerprint=d5eb6b1475d90ade6b322a4eea47b4e4a487f2546cbc29fd378f7743326dcdbe I0331 04:03:08.117512 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 243.113496ms to process 6 records W0331 04:03:08.905232 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0331 04:03:09.904774 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0331 04:03:10.338501 1 tasks_processing.go:74] worker 55 stopped. I0331 04:03:10.338618 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=895f64ae7b344d5ffc591abba1edfaf536b04cc72417168215dbced5e2d18241 I0331 04:03:10.338661 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=b1d3a57201f472fba248e6e22732d03f4ebac77f3d7204455295e9c6f31f376a I0331 04:03:10.338682 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=6d68fe02797e22400b823baf87ee367d453eff7f8a369561e197fd109848cd20 I0331 04:03:10.338692 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0331 04:03:10.338712 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=4ad5c115e1acb678644e6fc8ffca1399bda44830970e2674809f6d91801b8d45 I0331 04:03:10.338728 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0331 04:03:10.338750 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=42af0f4f685109f8c84550297c495a270f3ac339f5140d9a2c9025df3fc6b14e I0331 04:03:10.338781 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=d45e357c1710f49e0c4cf740f22cb402dd281418c2cd68956bb796587f7f0e84 I0331 04:03:10.338820 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=7889e9e98551023b511564d33bcd230563623ded273db76f90500838cd075aab I0331 04:03:10.338857 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=3367031930c6c5c5d5a03fb042727e9264710a699a97f9656167b4d5745c2780 I0331 04:03:10.339121 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=1a08aba2a5218d500d275ad45b6a1c708837cdd094d79179b33c0898ee9148a7 I0331 04:03:10.339140 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=030c2301622f8e72351e4ee51768265a8c025a2fb2d7478470b438d482d84eb7 I0331 04:03:10.339149 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0331 04:03:10.339164 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=f5c088503c76f85541415da668e41eb1afa5e5bbcbb11cf1d8bbc5644ecf9782 I0331 04:03:10.339174 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0331 04:03:10.339190 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=c72b25bf0e8cb6d01f23902930e6e2f610f19a50f0abf8722ba677bf4397b61f I0331 04:03:10.339200 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0331 04:03:10.339219 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=f0a402ad384dc2e856a8357b14856abfb68cd3e9c03a6d4eee9c986bdd2071df I0331 04:03:10.339228 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0331 04:03:10.339247 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=7185fd62e805caeccce9c4b368fd89ca80258003fb8185619b9327ea64afc78c I0331 04:03:10.339348 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=557a85b1c60b4137c645bf4efa9054be84a45754ca11e9c204ffb2ef786da7c2 I0331 04:03:10.339358 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0331 04:03:10.339364 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0331 04:03:10.339387 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0331 04:03:10.339408 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=3cc51f0c0bc07c41c89a117765a7813820b4b786af65178c8687dc3a75409dcb I0331 04:03:10.339431 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=88043846b4a9393cb32e6488bdf54696c0164265282b7760db40b341730514e3 I0331 04:03:10.339440 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0331 04:03:10.339457 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=92ef8e420058bbfd682ac1d231baed9613670b02617f71b7934dbb5d20b53f20 I0331 04:03:10.339467 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0331 04:03:10.339485 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=2ba011bdb4bc0110852fd578eb33b23aba19f47aab12019309158a39906992c1 I0331 04:03:10.339494 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0331 04:03:10.339516 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=52e50e7035573a92dcd04fa82830444d3e67ac9166c4fa9447a8213cad18e544 I0331 04:03:10.339535 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=e4e88e338b7fb3e7d2a80ad59eed2d9d95672b615b8e7be2228b625282bf7bc8 I0331 04:03:10.339551 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=7e3f2868bc4e7d04aa5cda6b8486272e20ee9a9e09b330282ddb7e94579ebecd I0331 04:03:10.339573 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=4310b3cf592289992d3aeafa94c4621bd4474ec9d2c502c7f2658959fa161b52 I0331 04:03:10.339582 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0331 04:03:10.339622 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=7bb2fc298f78deef5dd4418d74e4d9c93dde2fedfd76b87ab89842e43e9feb17 I0331 04:03:10.339641 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0331 04:03:10.339649 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0331 04:03:10.339656 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.466027489s to process 39 records W0331 04:03:10.905179 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0331 04:03:11.905020 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0331 04:03:12.903428 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded E0331 04:03:12.903454 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0331 04:03:12.903472 1 tasks_processing.go:74] worker 10 stopped. I0331 04:03:12.903477 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0331 04:03:12.903489 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.029345284s to process 1 records I0331 04:03:20.688181 1 tasks_processing.go:74] worker 39 stopped. I0331 04:03:20.688216 1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b I0331 04:03:20.688229 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.817000307s to process 1 records I0331 04:03:21.479891 1 tasks_processing.go:74] worker 34 stopped. I0331 04:03:21.480148 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=3f42842a95abf42f7e7c2b33333f34e223bd0ca1641d8e40233cb263a6e89e93 I0331 04:03:21.480164 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.606425108s to process 1 records E0331 04:03:21.480220 1 periodic.go:247] "Unhandled Error" err="clusterconfig failed after 13.61s with: function \"pod_network_connectivity_checks\" failed with an error, function \"active_alerts\" failed with an error, function \"silenced_alerts\" failed with an error, function \"support_secret\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"machines\" failed with an error, function \"tsdb_status\" failed with an error, function \"metrics\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0331 04:03:21.481330 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "pod_network_connectivity_checks" failed with an error, function "active_alerts" failed with an error, function "silenced_alerts" failed with an error, function "support_secret" failed with an error, function "machine_healthchecks" failed with an error, function "machines" failed with an error, function "tsdb_status" failed with an error, function "metrics" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0331 04:03:21.481344 1 periodic.go:209] Running workloads gatherer I0331 04:03:21.481360 1 tasks_processing.go:45] number of workers: 2 I0331 04:03:21.481368 1 tasks_processing.go:69] worker 1 listening for tasks. I0331 04:03:21.481372 1 tasks_processing.go:71] worker 1 working on workload_info task. I0331 04:03:21.481383 1 tasks_processing.go:69] worker 0 listening for tasks. I0331 04:03:21.481400 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0331 04:03:21.489258 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-149.ec2.internal... I0331 04:03:21.507538 1 tasks_processing.go:74] worker 0 stopped. I0331 04:03:21.507553 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 26.114026ms to process 0 records I0331 04:03:25.025878 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 4s I0331 04:03:25.039957 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 19s for image data I0331 04:03:25.049370 1 gather_workloads_info.go:387] No image sha256:2904a78e2eb73fd6a9bb94c105c2a056831fb4113fbb7b0607c50adc9d879c9b (10ms) I0331 04:03:25.060211 1 gather_workloads_info.go:387] No image sha256:765f0d23b637f685f98a31bd47c131b03cf72a40761a3f9a9d6320faa3c33733 (11ms) I0331 04:03:25.070117 1 gather_workloads_info.go:387] No image sha256:03cf4cd7ef1518610c6c7b3ad27d1622d82e98e3dc6e3f8e5d0fceb5c8d3786e (10ms) I0331 04:03:25.079901 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (10ms) I0331 04:03:25.088517 1 gather_workloads_info.go:387] No image sha256:5a95c19d82767e0235b4edb4a0536482c816904897aae1dc3eb255cb52b87a9f (9ms) I0331 04:03:25.096425 1 gather_workloads_info.go:387] No image sha256:3612c5a433effd17b7aff7ff6edf4fb27b61153908e3a0294158e4886a1d600d (8ms) I0331 04:03:25.104856 1 gather_workloads_info.go:387] No image sha256:a0105d1eb62cf6ac9e5e2ef28d3e89bf6dc514bc594fc7090fe5a5ee18a09c87 (8ms) I0331 04:03:25.112982 1 gather_workloads_info.go:387] No image sha256:a498046d64605bcccee2440aa4f04a4602baaae263cf01d977ec5208e876b1fd (8ms) I0331 04:03:25.121177 1 gather_workloads_info.go:387] No image sha256:1a2532940843248c57d52141185dd71fbc393ab28b65d48f682038632c1dbbad (8ms) I0331 04:03:25.131552 1 gather_workloads_info.go:398] Found image sha256:b3909bf664c77097f75b3768830863d642eed3815dab2bfb4415c771ca2d5007 (10ms) I0331 04:03:25.148635 1 gather_workloads_info.go:387] No image sha256:521712486e2c6e3c020dad6a1cb340db8e55665b69f7c208fab9cd9e965fd588 (17ms) I0331 04:03:25.252342 1 gather_workloads_info.go:387] No image sha256:a258c226562adb14e3a163a1940938526ee6a0928982a7667d85d9a7334ce639 (104ms) I0331 04:03:25.348799 1 gather_workloads_info.go:387] No image sha256:3d6ea9e1633f38070718a1357fcb80e6cb3932aafcc4d65e659795b186c5da15 (96ms) I0331 04:03:25.449596 1 gather_workloads_info.go:387] No image sha256:76cc47f3775560477ec548c7b6fddbd99112ad6604f790697eef546576051fb7 (101ms) I0331 04:03:25.549411 1 gather_workloads_info.go:387] No image sha256:0a99240166165eb5718e7516a43282fe32df9c7c5e809b31b58abe44e42ff94d (100ms) I0331 04:03:25.649937 1 gather_workloads_info.go:387] No image sha256:4556896f77307821531ef91b7b7faccb82b824ea695693b2989f597f0deca038 (101ms) I0331 04:03:25.749778 1 gather_workloads_info.go:387] No image sha256:50197f22710766515f67944a779e00dd9ae3d17b18732d7324a970353b11f292 (100ms) I0331 04:03:25.849604 1 gather_workloads_info.go:387] No image sha256:289816958633a763a72dbc44e1dad40466223164e7e253039514f0d974ea5d21 (100ms) I0331 04:03:25.949254 1 gather_workloads_info.go:387] No image sha256:36b9e89c3cfcf1ab9ae500486e38afb6862cba48cb0b4d84a09508ab8f3d299f (100ms) I0331 04:03:26.050005 1 gather_workloads_info.go:387] No image sha256:653c666f842c13e0baae2e89a9b1efe0e2ef56f621ffb5b32005115d2a26ab8c (101ms) I0331 04:03:26.149507 1 gather_workloads_info.go:387] No image sha256:695cf2f0cc07683c2a3ce1eaf3e56fe18abc6e2bac716f7d9843f5d173b9df52 (99ms) I0331 04:03:26.249905 1 gather_workloads_info.go:387] No image sha256:47154813651033d59751fb655a384dbffb64dd26f10bd7f3be0c3128d0486356 (100ms) I0331 04:03:26.349600 1 gather_workloads_info.go:387] No image sha256:943018739e3db1763c3184b460dbc409e058abbac76d57b9927faad317be85e4 (100ms) I0331 04:03:26.449233 1 gather_workloads_info.go:398] Found image sha256:32d4561d4bf1b3c4e3a7878f918f9c1422bab85bdf8711fbdf905d59692fee81 (100ms) I0331 04:03:26.550261 1 gather_workloads_info.go:387] No image sha256:2e564f336c77116053f34d4201d364d8da04e789cfffa0ea422574c95f2d6404 (101ms) I0331 04:03:26.649993 1 gather_workloads_info.go:387] No image sha256:2e57e192c3c1240fd935dcd55c8fde5e70e78bf81d6176c96edf21fafe59f8ba (100ms) I0331 04:03:26.749578 1 gather_workloads_info.go:387] No image sha256:91828234f107c068c8a4966d08370ae7b73e637651dbc6d92c18c4553402c22c (100ms) I0331 04:03:26.850211 1 gather_workloads_info.go:387] No image sha256:25bda9d34e23ba6bc9df5cf7104bdb237269740a3a76ee025f88b34e68b7e2b5 (101ms) I0331 04:03:26.949735 1 gather_workloads_info.go:387] No image sha256:2598489729a4b258e4ecda4a06f6875133f2a10ced5c5241f8a57a8a05418e36 (99ms) I0331 04:03:27.049296 1 gather_workloads_info.go:387] No image sha256:7adc1eab05d6724c76ba751f6df816b08d6e70b78dee9eb94fa6fd9690542c98 (100ms) I0331 04:03:27.149770 1 gather_workloads_info.go:387] No image sha256:c940ea87e7d133d75ba0002ef00c0806825eed3db8094cdb260d1bac18127733 (100ms) I0331 04:03:27.249392 1 gather_workloads_info.go:387] No image sha256:56a85660a445eced5c79a595a0eccf590087c5672d50f49d4c25ad52f9a44f04 (100ms) I0331 04:03:27.349265 1 gather_workloads_info.go:387] No image sha256:5f0b67cfbbc381243fb91ccc17345b56d05f4d717c667e8c644e5bf05633ba71 (100ms) I0331 04:03:27.449884 1 gather_workloads_info.go:387] No image sha256:7b31223098f08328f5ddea8e5b871dbbd5f5a61ec550e8956f66793c0c6031a9 (101ms) I0331 04:03:27.549214 1 gather_workloads_info.go:387] No image sha256:a56211d075aa43cbb491f669a5b2e46ee023dc95b7d51dbac28f463948c5ad61 (99ms) I0331 04:03:27.549245 1 tasks_processing.go:74] worker 1 stopped. I0331 04:03:27.549473 1 recorder.go:75] Recording config/workload_info with fingerprint=4e03c96a1f5810d98daf84d277cac279dba7251d12902d1a91dcccf911557463 I0331 04:03:27.549488 1 gather.go:177] gatherer "workloads" function "workload_info" took 6.067865456s to process 1 records I0331 04:03:27.549502 1 periodic.go:256] Periodic gather workloads completed in 6.068s I0331 04:03:27.549511 1 controllerstatus.go:80] name=periodic-workloads healthy=true reason= message= I0331 04:03:27.549516 1 periodic.go:209] Running conditional gatherer I0331 04:03:27.555081 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules I0331 04:03:28.618153 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer E0331 04:03:28.620597 1 conditional_gatherer.go:324] unable to update alerts cache: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?match%5B%5D=ALERTS%7Balertstate%3D%22firing%22%7D&query=ALERTS": dial tcp: lookup prometheus-k8s.openshift-monitoring.svc on 172.30.0.10:53: no such host I0331 04:03:28.620675 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0331 04:03:28.627112 1 conditional_gatherer.go:392] cluster version is '4.20.8' E0331 04:03:28.627126 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0331 04:03:28.627131 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0331 04:03:28.627134 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0331 04:03:28.627138 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0331 04:03:28.627141 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0331 04:03:28.627146 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0331 04:03:28.627149 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0331 04:03:28.627152 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0331 04:03:28.627154 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0331 04:03:28.627158 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0331 04:03:28.627171 1 tasks_processing.go:45] number of workers: 3 I0331 04:03:28.627180 1 tasks_processing.go:69] worker 2 listening for tasks. I0331 04:03:28.627183 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0331 04:03:28.627194 1 tasks_processing.go:69] worker 0 listening for tasks. I0331 04:03:28.627201 1 tasks_processing.go:69] worker 1 listening for tasks. I0331 04:03:28.627212 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0331 04:03:28.627215 1 tasks_processing.go:71] worker 1 working on conditional_gatherer_rules task. I0331 04:03:28.627223 1 tasks_processing.go:74] worker 0 stopped. I0331 04:03:28.627273 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0331 04:03:28.627287 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.408µs to process 1 records I0331 04:03:28.627362 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=9fafdcbf373f1f367f1d69551ea59114592cce1681189be0f3dde774e0530661 I0331 04:03:28.627370 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.676µs to process 1 records I0331 04:03:28.627376 1 tasks_processing.go:74] worker 1 stopped. I0331 04:03:28.627439 1 tasks_processing.go:74] worker 2 stopped. I0331 04:03:28.627451 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 250.742µs to process 0 records I0331 04:03:28.627469 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0331 04:03:28.627487 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0331 04:03:28.652268 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=19e3cf76944b12811a8f5760447605a3333ef35b5a3a35ba754753388af80580 I0331 04:03:28.652403 1 diskrecorder.go:70] Writing 105 records to /var/lib/insights-operator/insights-2026-03-31-040328.tar.gz I0331 04:03:28.659144 1 diskrecorder.go:51] Wrote 105 records to disk in 6ms I0331 04:03:28.659174 1 periodic.go:278] Gathering cluster info every 2h0m0s I0331 04:03:28.659190 1 periodic.go:279] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0331 04:04:37.870912 1 diskrecorder.go:223] Found files to send: insights-2026-03-31-040328.tar.gz I0331 04:04:37.870942 1 insightsuploader.go:150] Checking archives to upload periodically every 15m34.277292403s I0331 04:04:37.870950 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0331 04:04:37.881214 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0331 04:04:38.203735 1 requests.go:87] Successfully reported id=2026-03-31T04:04:37Z x-rh-insights-request-id=6770a9db56a34029910b789163afb48c, wrote=48847 I0331 04:04:38.203839 1 insightsuploader.go:187] Uploaded report successfully in 332.876708ms I0331 04:04:38.203870 1 controller.go:128] Initializing last reported time to 2026-03-31T04:04:37Z I0331 04:04:38.203888 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0331 04:04:38.203897 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0331 04:04:38.203904 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0331 04:04:38.210534 1 controller.go:512] The operator is healthy I0331 04:05:07.886676 1 controller.go:512] The operator is healthy I0331 04:05:44.056653 1 insightsreport.go:137] Pulling report from smart-proxy I0331 04:05:44.056687 1 insightsreport.go:149] Retrieving report I0331 04:05:44.063135 1 requests.go:110] Retrieving report for cluster: f5b165b4-3785-405a-85ec-9df281dd7456 I0331 04:05:44.063146 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/f5b165b4-3785-405a-85ec-9df281dd7456/reports I0331 04:05:44.065839 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/f5b165b4-3785-405a-85ec-9df281dd7456/reports I0331 04:05:44.220984 1 insightsreport.go:184] Report retrieved I0331 04:05:44.247970 1 insightsreport.go:239] Report retrieved correctly I0331 04:07:07.885832 1 controller.go:512] The operator is healthy I0331 04:08:07.864038 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0331 04:08:07.869639 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0331 04:08:07.869657 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0331 04:08:07.873737 1 secretconfigobserver.go:119] support secret does not exist I0331 04:09:07.885568 1 controller.go:512] The operator is healthy I0331 04:11:07.885096 1 controller.go:512] The operator is healthy I0331 04:13:07.860155 1 configmapobserver.go:84] configmaps "insights-config" not found I0331 04:13:07.864234 1 configmapobserver.go:84] configmaps "insights-config" not found I0331 04:13:07.874340 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0331 04:13:07.878756 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0331 04:13:07.878771 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0331 04:13:07.882566 1 secretconfigobserver.go:119] support secret does not exist I0331 04:13:07.884334 1 controller.go:512] The operator is healthy I0331 04:15:07.885664 1 controller.go:512] The operator is healthy I0331 04:17:07.885893 1 controller.go:512] The operator is healthy I0331 04:18:07.883416 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0331 04:18:07.889144 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0331 04:18:07.889162 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0331 04:18:07.893113 1 secretconfigobserver.go:119] support secret does not exist I0331 04:19:07.885570 1 controller.go:512] The operator is healthy I0331 04:20:17.482341 1 insightsuploader.go:146] Nothing to report since 2026-03-31T04:04:37Z I0331 04:21:07.885874 1 controller.go:512] The operator is healthy I0331 04:23:07.861159 1 configmapobserver.go:84] configmaps "insights-config" not found I0331 04:23:07.865619 1 configmapobserver.go:84] configmaps "insights-config" not found I0331 04:23:07.885852 1 controller.go:512] The operator is healthy I0331 04:23:07.893681 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0331 04:23:07.898557 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0331 04:23:07.898575 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0331 04:23:07.902654 1 secretconfigobserver.go:119] support secret does not exist I0331 04:25:07.885733 1 controller.go:512] The operator is healthy I0331 04:27:07.887989 1 controller.go:512] The operator is healthy I0331 04:28:07.903188 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0331 04:28:07.908968 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0331 04:28:07.908986 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0331 04:28:07.912786 1 secretconfigobserver.go:119] support secret does not exist I0331 04:29:07.886008 1 controller.go:512] The operator is healthy I0331 04:31:07.886020 1 controller.go:512] The operator is healthy I0331 04:33:07.861856 1 configmapobserver.go:84] configmaps "insights-config" not found I0331 04:33:07.866113 1 configmapobserver.go:84] configmaps "insights-config" not found I0331 04:33:07.885052 1 controller.go:512] The operator is healthy I0331 04:33:07.913037 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0331 04:33:07.929258 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0331 04:33:07.929278 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0331 04:33:07.933520 1 secretconfigobserver.go:119] support secret does not exist I0331 04:35:07.898723 1 controller.go:512] The operator is healthy I0331 04:36:25.254980 1 insightsuploader.go:146] Nothing to report since 2026-03-31T04:04:37Z I0331 04:37:07.886499 1 controller.go:512] The operator is healthy I0331 04:38:07.934420 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0331 04:38:07.940218 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0331 04:38:07.940239 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0331 04:38:07.944505 1 secretconfigobserver.go:119] support secret does not exist I0331 04:39:07.887987 1 controller.go:512] The operator is healthy I0331 04:41:07.885698 1 controller.go:512] The operator is healthy I0331 04:43:07.862012 1 configmapobserver.go:84] configmaps "insights-config" not found I0331 04:43:07.866217 1 configmapobserver.go:84] configmaps "insights-config" not found I0331 04:43:07.885600 1 controller.go:512] The operator is healthy I0331 04:43:07.944754 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0331 04:43:07.949619 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0331 04:43:07.949636 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0331 04:43:07.953610 1 secretconfigobserver.go:119] support secret does not exist