W0416 12:15:33.391287 1 cmd.go:257] Using insecure, self-signed certificates I0416 12:15:34.430721 1 start.go:138] Unable to read service ca bundle: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0416 12:15:34.431012 1 observer_polling.go:159] Starting file observer I0416 12:15:34.597102 1 operator.go:76] Starting insights-operator v0.0.0-master+$Format:%H$ I0416 12:15:34.597288 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0416 12:15:34.597890 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0416 12:15:34.597893 1 secure_serving.go:57] Forcing use of http/1.1 only W0416 12:15:34.597923 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0416 12:15:34.597930 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0416 12:15:34.597935 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0416 12:15:34.597939 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0416 12:15:34.597942 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0416 12:15:34.597945 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0416 12:15:34.601744 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"867391a5-7d97-46ac-8b81-8e5d04fde348", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "HyperShiftOnlyDynamicResourceAllocation", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryption", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLM", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "OpenShiftPodSecurityAdmission", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}} I0416 12:15:34.601685 1 operator.go:141] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNSInstall GatewayAPI GatewayAPIController HighlyAvailableArbiter HyperShiftOnlyDynamicResourceAllocation ImageStreamImportMode ImageVolume KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS ManagedBootImagesAzure ManagedBootImagesvSphere MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages PreconfiguredUDNAddresses ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SigstoreImageVerification SigstoreImageVerificationPKI StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSDualStackInstall AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureDualStackInstall AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement CBORServingAndStorage CRDCompatibilityRequirementOperator ClientsAllowCBOR ClientsPreferCBOR ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterAPIMachineManagement ClusterAPIMachineManagementVSphere ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud EtcdBackendQuota EventTTL EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall GCPDualStackInstall ImageModeStatusReporting IngressControllerDynamicConfigurationManager InsightsConfig InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryption KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesCPMS MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutableCSINodeAllocatableCount MutatingAdmissionPolicy NewOLM NewOLMBoxCutterRuntime NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterInstall NutanixMultiSubnets OSStreams OVNObservability OnPremDNSRecords OpenShiftPodSecurityAdmission ProvisioningRequestAvailable SELinuxMount ShortCertRotation SignatureStores TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeGroupSnapshot] I0416 12:15:34.602686 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0416 12:15:34.602731 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0416 12:15:34.602745 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0416 12:15:34.602747 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0416 12:15:34.602766 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0416 12:15:34.602770 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0416 12:15:34.603347 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/tmp/serving-cert-3236289026/tls.crt::/tmp/serving-cert-3236289026/tls.key" I0416 12:15:34.603720 1 secure_serving.go:213] Serving securely on [::]:8443 I0416 12:15:34.603748 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0416 12:15:34.610485 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0416 12:15:34.610506 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0416 12:15:34.610609 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0416 12:15:34.615004 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0416 12:15:34.615019 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0416 12:15:34.619416 1 secretconfigobserver.go:119] support secret does not exist I0416 12:15:34.623968 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0416 12:15:34.628137 1 secretconfigobserver.go:119] support secret does not exist I0416 12:15:34.630159 1 recorder.go:176] Pruning old reports every 5h1m5s, max age is 288h0m0s I0416 12:15:34.634890 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0416 12:15:34.634905 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0416 12:15:34.634907 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0416 12:15:34.634912 1 insightsreport.go:296] Starting report retriever I0416 12:15:34.634916 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0416 12:15:34.634892 1 periodic.go:216] Running clusterconfig gatherer I0416 12:15:34.634954 1 tasks_processing.go:45] number of workers: 64 I0416 12:15:34.634979 1 tasks_processing.go:69] worker 4 listening for tasks. I0416 12:15:34.634992 1 tasks_processing.go:69] worker 0 listening for tasks. I0416 12:15:34.634991 1 tasks_processing.go:69] worker 14 listening for tasks. I0416 12:15:34.634998 1 tasks_processing.go:69] worker 5 listening for tasks. I0416 12:15:34.634993 1 tasks_processing.go:69] worker 9 listening for tasks. I0416 12:15:34.635004 1 tasks_processing.go:69] worker 1 listening for tasks. I0416 12:15:34.635007 1 tasks_processing.go:69] worker 6 listening for tasks. I0416 12:15:34.635009 1 tasks_processing.go:69] worker 12 listening for tasks. I0416 12:15:34.635013 1 tasks_processing.go:69] worker 7 listening for tasks. I0416 12:15:34.635016 1 tasks_processing.go:69] worker 13 listening for tasks. I0416 12:15:34.635018 1 tasks_processing.go:71] worker 7 working on nodenetworkstates task. I0416 12:15:34.635018 1 tasks_processing.go:69] worker 11 listening for tasks. I0416 12:15:34.635019 1 tasks_processing.go:69] worker 3 listening for tasks. I0416 12:15:34.635024 1 tasks_processing.go:69] worker 28 listening for tasks. I0416 12:15:34.635025 1 tasks_processing.go:69] worker 43 listening for tasks. I0416 12:15:34.635031 1 tasks_processing.go:69] worker 58 listening for tasks. I0416 12:15:34.635033 1 tasks_processing.go:69] worker 10 listening for tasks. I0416 12:15:34.635033 1 tasks_processing.go:69] worker 42 listening for tasks. I0416 12:15:34.635039 1 tasks_processing.go:69] worker 17 listening for tasks. I0416 12:15:34.635041 1 tasks_processing.go:69] worker 35 listening for tasks. I0416 12:15:34.635044 1 tasks_processing.go:69] worker 31 listening for tasks. I0416 12:15:34.635028 1 tasks_processing.go:69] worker 15 listening for tasks. I0416 12:15:34.635034 1 tasks_processing.go:69] worker 16 listening for tasks. I0416 12:15:34.635053 1 tasks_processing.go:69] worker 19 listening for tasks. I0416 12:15:34.635038 1 tasks_processing.go:69] worker 59 listening for tasks. I0416 12:15:34.635042 1 tasks_processing.go:69] worker 57 listening for tasks. I0416 12:15:34.635056 1 tasks_processing.go:69] worker 41 listening for tasks. I0416 12:15:34.635060 1 tasks_processing.go:69] worker 33 listening for tasks. I0416 12:15:34.635064 1 tasks_processing.go:69] worker 21 listening for tasks. I0416 12:15:34.635061 1 tasks_processing.go:69] worker 38 listening for tasks. I0416 12:15:34.635070 1 tasks_processing.go:69] worker 8 listening for tasks. I0416 12:15:34.635072 1 tasks_processing.go:69] worker 34 listening for tasks. I0416 12:15:34.635074 1 tasks_processing.go:71] worker 9 working on machine_config_pools task. I0416 12:15:34.635074 1 tasks_processing.go:69] worker 30 listening for tasks. I0416 12:15:34.635078 1 tasks_processing.go:71] worker 1 working on image_registries task. I0416 12:15:34.635081 1 tasks_processing.go:71] worker 6 working on version task. I0416 12:15:34.635081 1 tasks_processing.go:69] worker 37 listening for tasks. I0416 12:15:34.635081 1 tasks_processing.go:69] worker 23 listening for tasks. I0416 12:15:34.635089 1 tasks_processing.go:69] worker 27 listening for tasks. I0416 12:15:34.635091 1 tasks_processing.go:69] worker 53 listening for tasks. I0416 12:15:34.635094 1 tasks_processing.go:69] worker 24 listening for tasks. I0416 12:15:34.635062 1 tasks_processing.go:71] worker 4 working on metrics task. I0416 12:15:34.635099 1 tasks_processing.go:69] worker 49 listening for tasks. I0416 12:15:34.635103 1 tasks_processing.go:69] worker 26 listening for tasks. I0416 12:15:34.635107 1 tasks_processing.go:69] worker 51 listening for tasks. I0416 12:15:34.635111 1 tasks_processing.go:71] worker 13 working on validating_webhook_configurations task. I0416 12:15:34.635114 1 tasks_processing.go:71] worker 26 working on container_images task. I0416 12:15:34.635117 1 tasks_processing.go:71] worker 51 working on networks task. I0416 12:15:34.635119 1 tasks_processing.go:71] worker 41 working on machine_autoscalers task. I0416 12:15:34.635123 1 tasks_processing.go:71] worker 3 working on nodes task. W0416 12:15:34.635123 1 gather_most_recent_metrics.go:64] Unable to load metrics client, no metrics will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0416 12:15:34.635125 1 tasks_processing.go:71] worker 17 working on operators task. I0416 12:15:34.635129 1 tasks_processing.go:69] worker 55 listening for tasks. I0416 12:15:34.635138 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 27.743µs to process 0 records I0416 12:15:34.635140 1 tasks_processing.go:71] worker 43 working on qemu_kubevirt_launcher_logs task. I0416 12:15:34.635150 1 tasks_processing.go:71] worker 58 working on service_accounts task. I0416 12:15:34.635160 1 tasks_processing.go:71] worker 21 working on sap_config task. I0416 12:15:34.635114 1 tasks_processing.go:71] worker 57 working on certificate_signing_requests task. I0416 12:15:34.635163 1 tasks_processing.go:71] worker 23 working on machines task. I0416 12:15:34.635170 1 tasks_processing.go:71] worker 27 working on node_logs task. I0416 12:15:34.635119 1 tasks_processing.go:71] worker 11 working on aggregated_monitoring_cr_names task. I0416 12:15:34.635050 1 tasks_processing.go:69] worker 44 listening for tasks. I0416 12:15:34.635074 1 tasks_processing.go:69] worker 22 listening for tasks. I0416 12:15:34.635104 1 tasks_processing.go:71] worker 24 working on dvo_metrics task. I0416 12:15:34.635484 1 tasks_processing.go:71] worker 44 working on install_plans task. I0416 12:15:34.635117 1 tasks_processing.go:71] worker 35 working on jaegers task. I0416 12:15:34.635069 1 tasks_processing.go:69] worker 40 listening for tasks. I0416 12:15:34.635151 1 tasks_processing.go:71] worker 34 working on active_alerts task. I0416 12:15:34.636330 1 tasks_processing.go:71] worker 40 working on image_pruners task. W0416 12:15:34.636392 1 gather_active_alerts.go:54] Unable to load alerts client, no alerts will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0416 12:15:34.635019 1 tasks_processing.go:71] worker 12 working on pod_network_connectivity_checks task. I0416 12:15:34.635054 1 tasks_processing.go:69] worker 32 listening for tasks. I0416 12:15:34.635054 1 tasks_processing.go:69] worker 62 listening for tasks. I0416 12:15:34.635042 1 tasks_processing.go:69] worker 60 listening for tasks. I0416 12:15:34.635059 1 tasks_processing.go:69] worker 63 listening for tasks. I0416 12:15:34.635060 1 tasks_processing.go:69] worker 20 listening for tasks. I0416 12:15:34.635060 1 tasks_processing.go:69] worker 45 listening for tasks. I0416 12:15:34.635048 1 tasks_processing.go:69] worker 18 listening for tasks. I0416 12:15:34.635013 1 tasks_processing.go:69] worker 2 listening for tasks. I0416 12:15:34.635065 1 tasks_processing.go:71] worker 0 working on storage_cluster task. I0416 12:15:34.635067 1 tasks_processing.go:69] worker 29 listening for tasks. I0416 12:15:34.635068 1 tasks_processing.go:71] worker 14 working on clusterroles task. I0416 12:15:34.635067 1 tasks_processing.go:69] worker 46 listening for tasks. I0416 12:15:34.635071 1 tasks_processing.go:71] worker 5 working on machine_configs task. I0416 12:15:34.635049 1 tasks_processing.go:69] worker 39 listening for tasks. I0416 12:15:34.635075 1 tasks_processing.go:69] worker 47 listening for tasks. I0416 12:15:34.635083 1 tasks_processing.go:69] worker 48 listening for tasks. I0416 12:15:34.635087 1 tasks_processing.go:69] worker 25 listening for tasks. I0416 12:15:34.635083 1 tasks_processing.go:69] worker 36 listening for tasks. I0416 12:15:34.635096 1 tasks_processing.go:69] worker 52 listening for tasks. I0416 12:15:34.636454 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 94.691µs to process 0 records I0416 12:15:34.635101 1 tasks_processing.go:69] worker 54 listening for tasks. I0416 12:15:34.635106 1 tasks_processing.go:71] worker 19 working on storage_classes task. I0416 12:15:34.636515 1 tasks_processing.go:71] worker 54 working on authentication task. I0416 12:15:34.636618 1 tasks_processing.go:71] worker 39 working on ingress_certificates task. I0416 12:15:34.636953 1 tasks_processing.go:71] worker 63 working on config_maps task. I0416 12:15:34.636990 1 tasks_processing.go:71] worker 45 working on pdbs task. I0416 12:15:34.635107 1 tasks_processing.go:71] worker 49 working on sap_datahubs task. I0416 12:15:34.636824 1 tasks_processing.go:71] worker 20 working on lokistack task. I0416 12:15:34.637183 1 tasks_processing.go:71] worker 18 working on openstack_dataplanedeployments task. I0416 12:15:34.635110 1 tasks_processing.go:71] worker 59 working on monitoring_persistent_volumes task. I0416 12:15:34.637438 1 tasks_processing.go:71] worker 2 working on openshift_logging task. I0416 12:15:34.637487 1 tasks_processing.go:71] worker 52 working on crds task. I0416 12:15:34.635108 1 tasks_processing.go:69] worker 56 listening for tasks. I0416 12:15:34.637450 1 tasks_processing.go:71] worker 46 working on machine_sets task. I0416 12:15:34.635115 1 tasks_processing.go:71] worker 10 working on operators_pods_and_events task. I0416 12:15:34.635121 1 tasks_processing.go:71] worker 42 working on ceph_cluster task. I0416 12:15:34.635155 1 tasks_processing.go:71] worker 33 working on schedulers task. I0416 12:15:34.635240 1 tasks_processing.go:71] worker 53 working on infrastructures task. I0416 12:15:34.635545 1 tasks_processing.go:71] worker 55 working on openshift_machine_api_events task. I0416 12:15:34.635589 1 tasks_processing.go:71] worker 38 working on oauths task. I0416 12:15:34.635604 1 tasks_processing.go:71] worker 15 working on machine_healthchecks task. I0416 12:15:34.635611 1 tasks_processing.go:71] worker 31 working on overlapping_namespace_uids task. I0416 12:15:34.635619 1 tasks_processing.go:71] worker 37 working on cost_management_metrics_configs task. I0416 12:15:34.635625 1 tasks_processing.go:71] worker 30 working on nodenetworkconfigurationpolicies task. I0416 12:15:34.635655 1 tasks_processing.go:71] worker 28 working on feature_gates task. I0416 12:15:34.635667 1 tasks_processing.go:71] worker 4 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0416 12:15:34.635704 1 tasks_processing.go:71] worker 22 working on mutating_webhook_configurations task. I0416 12:15:34.635720 1 tasks_processing.go:71] worker 16 working on openstack_dataplanenodesets task. I0416 12:15:34.635049 1 tasks_processing.go:69] worker 61 listening for tasks. I0416 12:15:34.635781 1 tasks_processing.go:71] worker 8 working on container_runtime_configs task. I0416 12:15:34.635099 1 tasks_processing.go:69] worker 50 listening for tasks. I0416 12:15:34.638862 1 tasks_processing.go:71] worker 50 working on support_secret task. I0416 12:15:34.637448 1 tasks_processing.go:71] worker 34 working on node_features task. I0416 12:15:34.637456 1 tasks_processing.go:71] worker 47 working on tsdb_status task. I0416 12:15:34.637460 1 tasks_processing.go:71] worker 32 working on olm_operators task. W0416 12:15:34.638981 1 gather_prometheus_tsdb_status.go:38] Unable to load metrics client, tsdb status cannot be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0416 12:15:34.638990 1 tasks_processing.go:74] worker 47 stopped. I0416 12:15:34.638998 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 21.043µs to process 0 records I0416 12:15:34.637461 1 tasks_processing.go:71] worker 62 working on openstack_version task. I0416 12:15:34.637464 1 tasks_processing.go:71] worker 48 working on cluster_apiserver task. I0416 12:15:34.637466 1 tasks_processing.go:71] worker 29 working on sap_pods task. I0416 12:15:34.637467 1 tasks_processing.go:71] worker 25 working on silenced_alerts task. W0416 12:15:34.639164 1 gather_silenced_alerts.go:38] Unable to load alerts client, no alerts will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0416 12:15:34.637471 1 tasks_processing.go:71] worker 36 working on ingress task. I0416 12:15:34.639178 1 tasks_processing.go:74] worker 25 stopped. I0416 12:15:34.639190 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 31.975µs to process 0 records I0416 12:15:34.637472 1 tasks_processing.go:71] worker 60 working on openstack_controlplanes task. I0416 12:15:34.637616 1 tasks_processing.go:71] worker 56 working on proxies task. I0416 12:15:34.639324 1 tasks_processing.go:74] worker 9 stopped. I0416 12:15:34.639341 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 4.240896ms to process 0 records I0416 12:15:34.639363 1 tasks_processing.go:74] worker 7 stopped. I0416 12:15:34.638813 1 tasks_processing.go:71] worker 61 working on image task. I0416 12:15:34.639384 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 4.339526ms to process 0 records I0416 12:15:34.639424 1 tasks_processing.go:74] worker 41 stopped. I0416 12:15:34.639444 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 4.291176ms to process 0 records I0416 12:15:34.640203 1 tasks_processing.go:74] worker 21 stopped. I0416 12:15:34.640216 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 5.03464ms to process 0 records I0416 12:15:34.640430 1 controller.go:129] Initializing last reported time to 0001-01-01T00:00:00Z I0416 12:15:34.640462 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0416 12:15:34.640467 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0416 12:15:34.640470 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0416 12:15:34.640483 1 controller.go:531] The operator is still being initialized I0416 12:15:34.640491 1 controller.go:554] The operator is healthy I0416 12:15:34.640727 1 tasks_processing.go:74] worker 23 stopped. E0416 12:15:34.640738 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0416 12:15:34.640745 1 gather.go:177] gatherer "clusterconfig" function "machines" took 5.54434ms to process 0 records I0416 12:15:34.641854 1 tasks_processing.go:74] worker 51 stopped. I0416 12:15:34.642058 1 recorder.go:75] Recording config/network with fingerprint=4200a474b57baa8812e3ce67f66f18035ff0637e6814e373f8a7b96366254a71 I0416 12:15:34.642074 1 gather.go:177] gatherer "clusterconfig" function "networks" took 6.724213ms to process 1 records I0416 12:15:34.644031 1 tasks_processing.go:74] worker 0 stopped. I0416 12:15:34.644041 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 7.560527ms to process 0 records I0416 12:15:34.644358 1 tasks_processing.go:74] worker 12 stopped. E0416 12:15:34.644369 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0416 12:15:34.644379 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 7.377388ms to process 0 records I0416 12:15:34.657710 1 tasks_processing.go:74] worker 49 stopped. I0416 12:15:34.657719 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 20.703187ms to process 0 records I0416 12:15:34.657727 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 20.655389ms to process 0 records I0416 12:15:34.657731 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 21.797685ms to process 0 records I0416 12:15:34.657736 1 tasks_processing.go:74] worker 35 stopped. I0416 12:15:34.657740 1 tasks_processing.go:74] worker 20 stopped. I0416 12:15:34.660864 1 tasks_processing.go:74] worker 2 stopped. I0416 12:15:34.660881 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 23.371341ms to process 0 records I0416 12:15:34.660888 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 22.737847ms to process 0 records I0416 12:15:34.660893 1 tasks_processing.go:74] worker 37 stopped. I0416 12:15:34.660917 1 tasks_processing.go:74] worker 15 stopped. E0416 12:15:34.660930 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0416 12:15:34.660937 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 23.009328ms to process 0 records I0416 12:15:34.660943 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 23.173081ms to process 0 records I0416 12:15:34.660950 1 tasks_processing.go:74] worker 42 stopped. I0416 12:15:34.661060 1 gather_logs.go:145] no pods in namespace were found I0416 12:15:34.661073 1 tasks_processing.go:74] worker 43 stopped. I0416 12:15:34.661082 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 25.917278ms to process 0 records I0416 12:15:34.661839 1 tasks_processing.go:74] worker 30 stopped. I0416 12:15:34.661849 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 23.368611ms to process 0 records I0416 12:15:34.661856 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 24.225022ms to process 0 records I0416 12:15:34.661862 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 23.073489ms to process 0 records I0416 12:15:34.661864 1 tasks_processing.go:74] worker 46 stopped. I0416 12:15:34.661869 1 tasks_processing.go:74] worker 16 stopped. I0416 12:15:34.661890 1 tasks_processing.go:74] worker 18 stopped. I0416 12:15:34.661907 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 24.649029ms to process 0 records I0416 12:15:34.661916 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 24.103854ms to process 0 records I0416 12:15:34.661923 1 tasks_processing.go:74] worker 55 stopped. I0416 12:15:34.661997 1 tasks_processing.go:74] worker 62 stopped. I0416 12:15:34.662007 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 22.985288ms to process 0 records E0416 12:15:34.662021 1 gather_node_features.go:86] GatherNodeFeatures: NodeFeatures resource not found in openshift-nfd namespace (may not be installed) I0416 12:15:34.662026 1 tasks_processing.go:74] worker 34 stopped. I0416 12:15:34.662033 1 gather.go:177] gatherer "clusterconfig" function "node_features" took 23.084548ms to process 0 records I0416 12:15:34.662065 1 tasks_processing.go:74] worker 8 stopped. I0416 12:15:34.662073 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 23.243547ms to process 0 records I0416 12:15:34.662216 1 tasks_processing.go:74] worker 38 stopped. I0416 12:15:34.662589 1 recorder.go:75] Recording config/oauth with fingerprint=ce37728f35c97c788fe8ba8b63e9dba55573f7878bd0ddb86377cdf8243400c7 I0416 12:15:34.662611 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 24.335313ms to process 1 records I0416 12:15:34.662783 1 tasks_processing.go:74] worker 45 stopped. I0416 12:15:34.662788 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=9106fe55bb4bbc6929b80077556411cfb5e4d0e25594376348932eb51b8d0e56 I0416 12:15:34.662854 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=ea502d6e121497b88da186fdbc029fccfc5cfa7860f217b545635d43482cadbc I0416 12:15:34.662884 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=7e811df452dafb9d705725f81ea8a96b9a3e2fffbe0c099514b5492b3811dbe5 I0416 12:15:34.662892 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 25.290232ms to process 3 records I0416 12:15:34.662920 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 23.072573ms to process 0 records E0416 12:15:34.662933 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0416 12:15:34.662942 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 23.52265ms to process 0 records I0416 12:15:34.662977 1 tasks_processing.go:74] worker 60 stopped. I0416 12:15:34.662998 1 tasks_processing.go:74] worker 50 stopped. I0416 12:15:34.663132 1 tasks_processing.go:74] worker 54 stopped. I0416 12:15:34.663802 1 recorder.go:75] Recording config/authentication with fingerprint=7fd7c77f7aaa9b812164ac5942a95bb7f66a9f927339fd49fd503e5bac7f6a2e I0416 12:15:34.663855 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 25.816425ms to process 1 records I0416 12:15:34.663902 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 23.300471ms to process 0 records I0416 12:15:34.663950 1 tasks_processing.go:74] worker 29 stopped. I0416 12:15:34.663984 1 tasks_processing.go:74] worker 53 stopped. I0416 12:15:34.665772 1 recorder.go:75] Recording config/infrastructure with fingerprint=21a901e37842e68aa5c6a62f96d8e2c3a100f0059f9546f04351f7ce42dee7f3 I0416 12:15:34.665896 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 24.760802ms to process 1 records I0416 12:15:34.666044 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=92b4fff77fa92671c68efeea8bf71aea0f01479fc8eeb2f7e14881dee8dbda58 I0416 12:15:34.666077 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 24.964279ms to process 1 records I0416 12:15:34.666108 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 30.201096ms to process 0 records I0416 12:15:34.666138 1 tasks_processing.go:74] worker 27 stopped. I0416 12:15:34.666152 1 tasks_processing.go:74] worker 33 stopped. I0416 12:15:34.666428 1 tasks_processing.go:74] worker 36 stopped. I0416 12:15:34.667180 1 recorder.go:75] Recording config/ingress with fingerprint=ffd7d8c872a943f7b88337df1dd5ec866186028be4126d37267bb6950b5962df I0416 12:15:34.667588 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 27.245164ms to process 1 records I0416 12:15:34.667714 1 tasks_processing.go:74] worker 19 stopped. I0416 12:15:34.668621 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=5c2c8be3b491a612094a7374892d4facfa9074b7881b91bd747fd61c2f95faaa I0416 12:15:34.668653 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=8e339fc5571e189a40f69532dfc4b26e6983b121af369401a5707361a15f24b4 I0416 12:15:34.668663 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 30.115488ms to process 2 records I0416 12:15:34.668774 1 tasks_processing.go:74] worker 40 stopped. I0416 12:15:34.668950 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=501f15afd994ec95f94f6384efcb2bcdd3c410131170b0b9f392d1947f6dea7a I0416 12:15:34.668964 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 30.357453ms to process 1 records I0416 12:15:34.668971 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 29.476732ms to process 0 records I0416 12:15:34.669002 1 tasks_processing.go:74] worker 59 stopped. I0416 12:15:34.669035 1 recorder.go:75] Recording config/proxy with fingerprint=3dc36ab7ec20ba5cc3222922b3deb038c1b1f7a365e0b935c9d267b09707ccc6 I0416 12:15:34.669085 1 tasks_processing.go:74] worker 56 stopped. I0416 12:15:34.669088 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 27.747981ms to process 1 records E0416 12:15:34.669128 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0416 12:15:34.669184 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0416 12:15:34.669202 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 30.022223ms to process 1 records I0416 12:15:34.669211 1 tasks_processing.go:74] worker 5 stopped. I0416 12:15:34.669347 1 tasks_processing.go:74] worker 22 stopped. I0416 12:15:34.669514 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=5f8435efb9d8f71554beddcf053d28e00ed024269b7460b6a5f2a82c684ababa I0416 12:15:34.669584 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=49c8f28ea30648311fc036c5e7b23511bbf4e4f0d517813f6d22b9a259a3fe13 I0416 12:15:34.669626 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=21429e4f23a01c882b566d102ef7d95c48cc7e4e228747a8c9c5c5296809aeb5 I0416 12:15:34.669638 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 29.541817ms to process 3 records I0416 12:15:34.669721 1 recorder.go:75] Recording config/image with fingerprint=d4c6d607548a9d62a96c5173e2328b93129c2a0b08f80c115ffa80b64c17f5ab I0416 12:15:34.669732 1 gather.go:177] gatherer "clusterconfig" function "image" took 28.986479ms to process 1 records I0416 12:15:34.669848 1 tasks_processing.go:74] worker 61 stopped. I0416 12:15:34.669909 1 tasks_processing.go:74] worker 1 stopped. W0416 12:15:34.670044 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0416 12:15:34.670118 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=5e0283f22d2d6197a2356474fc1002250ab114d0c90cfe1045f13a35ce2d6958 I0416 12:15:34.670134 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 33.4984ms to process 1 records I0416 12:15:34.670249 1 tasks_processing.go:74] worker 3 stopped. I0416 12:15:34.670407 1 recorder.go:75] Recording config/node/ip-10-0-0-136.ec2.internal with fingerprint=b263555a3ac7b001f50273e0265cb1219168cfea7fab02d2db628efce3a48a2d I0416 12:15:34.670476 1 recorder.go:75] Recording config/node/ip-10-0-1-109.ec2.internal with fingerprint=d14b26d3f9a547e4edfe35184e059a52f28d15db6836452727a0a70937096132 I0416 12:15:34.670581 1 recorder.go:75] Recording config/node/ip-10-0-2-126.ec2.internal with fingerprint=9076d0a20c32be7f393ae0d3faf6c97de8802848519d0a4cf0359a3c6041bd7a I0416 12:15:34.670594 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 33.534103ms to process 3 records I0416 12:15:34.670684 1 recorder.go:75] Recording config/featuregate with fingerprint=dc4552974dd1b1a16d33297632b6f99ab5c568962d8b47b56ca2135e2bf008c8 I0416 12:15:34.670694 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 30.0735ms to process 1 records I0416 12:15:34.670727 1 tasks_processing.go:74] worker 28 stopped. I0416 12:15:34.670807 1 recorder.go:75] Recording config/apiserver with fingerprint=13102a2c5eb684fd386d94237c95a25dc5cf39df875cbcbd4467f75e9369fd1b I0416 12:15:34.670813 1 tasks_processing.go:74] worker 48 stopped. I0416 12:15:34.670818 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 29.611228ms to process 1 records I0416 12:15:34.670904 1 tasks_processing.go:74] worker 13 stopped. I0416 12:15:34.670920 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=42cc20a7ab2eaef933717e8585ad56271ccbef165ec81a0b99f1b3c52d72c489 I0416 12:15:34.671007 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=878f0f2d7edcad7f250045ef3e35a3a2d366f4b1c0e3f87b345e682be2f116fe I0416 12:15:34.671036 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=aeda38bc24d156cd0ae63cfc88dec8413498253b985b9078eb8cd5178b19ec9f I0416 12:15:34.671071 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=a3e3ff4c67f6512b3f4e85862b123c25cb149928296f456f943f9ee3eae8a322 I0416 12:15:34.671103 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=de69056929a5aa8a531a074a1ad6abe0fcf19d4f8f6a8ce1ee71435e612dc257 I0416 12:15:34.671140 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=6fa8c7786ce00231c6c24f2cce750f2c97ed6c506fc9cf18c952515667ed6bf5 I0416 12:15:34.671188 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=51cee72ea8a1e2c2519fefc3e91d7710653573e261dfb63b8aa552c2848fe8d1 I0416 12:15:34.671233 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=bb959bf2fefd7a30d29b808fc19250a28fa7f35b69668f3f3a30280918d37e47 I0416 12:15:34.671266 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=83db4e45bebbedc5df24bf441e94bc755a5bab0a940749f33d06d0b39f5176e5 I0416 12:15:34.671298 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=a7f099286af3f3826ba01043744ba0c63856e615631532dd84891f4578cdeeb3 I0416 12:15:34.671329 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=25eb1f00c02e4be4ae8b331c35b8ce2c3d18b04b325d41225d9908cd198ad316 I0416 12:15:34.671341 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 34.501536ms to process 11 records I0416 12:15:34.671361 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0416 12:15:34.671372 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 31.961925ms to process 1 records I0416 12:15:34.671379 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 35.550416ms to process 0 records I0416 12:15:34.671389 1 tasks_processing.go:74] worker 57 stopped. I0416 12:15:34.671395 1 tasks_processing.go:74] worker 31 stopped. I0416 12:15:34.679754 1 tasks_processing.go:74] worker 14 stopped. I0416 12:15:34.679824 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0416 12:15:34.679887 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0416 12:15:34.679935 1 operator.go:328] started I0416 12:15:34.679956 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0416 12:15:34.680005 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=ebd429b4f1ff0897030d0106965820eaa5b351b0ba9ff2f32b83636e67e7f8a9 I0416 12:15:34.680110 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=09dfc2aecf9584ec541c9aedade78b01abd2c15ad92c9368e15a83b7be6e513c I0416 12:15:34.680120 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 42.928657ms to process 2 records I0416 12:15:34.680900 1 tasks_processing.go:74] worker 52 stopped. I0416 12:15:34.681685 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=7cacf1e3b710562e5bf90e57e3cd6301b224b260c6d1d3f149d9048198827afe I0416 12:15:34.682029 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=0111c3c55849a4123b7c5378617ba5e93b7e2bbba4a7d3d945fdc46f464a0949 I0416 12:15:34.682047 1 gather.go:177] gatherer "clusterconfig" function "crds" took 43.352427ms to process 2 records I0416 12:15:34.682063 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 45.900954ms to process 0 records I0416 12:15:34.682073 1 tasks_processing.go:74] worker 11 stopped. I0416 12:15:34.684140 1 tasks_processing.go:74] worker 26 stopped. I0416 12:15:34.684221 1 recorder.go:75] Recording config/running_containers with fingerprint=e0c34ed929182646422e7691f2272e99e234641aafb83559a164827d1e8c8ed8 I0416 12:15:34.684237 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 49.010081ms to process 1 records I0416 12:15:34.689101 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0416 12:15:34.689114 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0416 12:15:34.689118 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0416 12:15:34.689121 1 controller.go:254] Source scaController *sca.Controller is not ready I0416 12:15:34.689124 1 controller.go:254] Source clusterTransferController *clustertransfer.Controller is not ready I0416 12:15:34.689140 1 controller.go:531] The operator is still being initialized I0416 12:15:34.689147 1 controller.go:554] The operator is healthy I0416 12:15:34.690306 1 prometheus_rules.go:88] Prometheus rules successfully created I0416 12:15:34.690633 1 tasks_processing.go:74] worker 4 stopped. I0416 12:15:34.690649 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 51.952035ms to process 0 records I0416 12:15:34.694206 1 tasks_processing.go:74] worker 63 stopped. E0416 12:15:34.694222 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0416 12:15:34.694228 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0416 12:15:34.694231 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0416 12:15:34.694240 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=ad286723f58bdcfc37aeba3ec5b4110c08e1af59cd34d14b4bfaab02d18e4856 I0416 12:15:34.694279 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0416 12:15:34.694288 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0416 12:15:34.694294 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=2ab5ab7b1b10d7fcf1197bb24dea7c90f400e4effc18c7356873209d54fdf84b I0416 12:15:34.694300 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0416 12:15:34.694356 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0416 12:15:34.694369 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0416 12:15:34.694378 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 57.228457ms to process 7 records E0416 12:15:34.695881 1 cluster_transfer.go:95] failed to pull cluster transfer: unable to retrieve cluster transfer data from https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/: Get "https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/?search=cluster_uuid+is+%2707cb04e3-d071-4f65-bc3b-44d78e1079cd%27+and+status+is+%27accepted%27": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.20:46592->172.30.0.10:53: read: connection refused I0416 12:15:34.695897 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=Disconnected message=failed to pull cluster transfer: unable to retrieve cluster transfer data from https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/: Get "https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/?search=cluster_uuid+is+%2707cb04e3-d071-4f65-bc3b-44d78e1079cd%27+and+status+is+%27accepted%27": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.20:46592->172.30.0.10:53: read: connection refused I0416 12:15:34.703263 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0416 12:15:34.703330 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0416 12:15:34.703406 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0416 12:15:34.710972 1 base_controller.go:82] Caches are synced for ConfigController I0416 12:15:34.710984 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0416 12:15:34.723960 1 configmapobserver.go:84] configmaps "insights-config" not found I0416 12:15:34.748545 1 tasks_processing.go:74] worker 6 stopped. I0416 12:15:34.748769 1 recorder.go:75] Recording config/version with fingerprint=eb140bac0290719acd7bbe95a527d49734f835c229fe03e5a1687ad0d34d668b I0416 12:15:34.748781 1 recorder.go:75] Recording config/id with fingerprint=388860dd8c84bfc82898c92903729d9baa2c8fa7e82b36c753639c4b41c36e57 I0416 12:15:34.748787 1 gather.go:177] gatherer "clusterconfig" function "version" took 113.453148ms to process 2 records I0416 12:15:34.768046 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload W0416 12:15:34.771622 1 sca.go:161] Failed to pull SCA certs from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: unable to retrieve SCA certs data from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: Post "https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.20:59299->172.30.0.10:53: read: connection refused I0416 12:15:34.771634 1 controllerstatus.go:80] name=scaController healthy=true reason=NonHTTPError message=Failed to pull SCA certs from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: unable to retrieve SCA certs data from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: Post "https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.20:59299->172.30.0.10:53: read: connection refused I0416 12:15:34.780826 1 base_controller.go:82] Caches are synced for LoggingSyncer I0416 12:15:34.780836 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0416 12:15:34.785807 1 tasks_processing.go:74] worker 39 stopped. E0416 12:15:34.785824 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0416 12:15:34.785834 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2pn82so64dbfd7cg358bevtr5m7peubm-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2pn82so64dbfd7cg358bevtr5m7peubm-primary-cert-bundle-secret" not found I0416 12:15:34.785887 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=8a3b89d2235946e169b418453781a559804501cb07bd78483e0cbbe0a7f0d343 I0416 12:15:34.785900 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 149.148211ms to process 1 records I0416 12:15:35.075703 1 gather_cluster_operator_pods_and_events.go:121] Found 14 pods with 15 containers I0416 12:15:35.075716 1 gather_cluster_operator_pods_and_events.go:235] Maximum buffer size: 1677721 bytes I0416 12:15:35.076864 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns container dns-default-xt8sp pod in namespace openshift-dns (previous: false). I0416 12:15:35.281010 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-xt8sp pod in namespace openshift-dns for failing operator dns (previous: false): "container \"dns\" in pod \"dns-default-xt8sp\" is waiting to start: ContainerCreating" I0416 12:15:35.281026 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"dns\" in pod \"dns-default-xt8sp\" is waiting to start: ContainerCreating" I0416 12:15:35.281037 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-rbac-proxy container dns-default-xt8sp pod in namespace openshift-dns (previous: false). I0416 12:15:35.480233 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-xt8sp pod in namespace openshift-dns for failing operator kube-rbac-proxy (previous: false): "container \"kube-rbac-proxy\" in pod \"dns-default-xt8sp\" is waiting to start: ContainerCreating" I0416 12:15:35.480250 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"kube-rbac-proxy\" in pod \"dns-default-xt8sp\" is waiting to start: ContainerCreating" I0416 12:15:35.480261 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-8lnxh pod in namespace openshift-dns (previous: false). W0416 12:15:35.672179 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0416 12:15:35.681250 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0416 12:15:35.681263 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-frg42 pod in namespace openshift-dns (previous: false). I0416 12:15:35.686885 1 tasks_processing.go:74] worker 32 stopped. I0416 12:15:35.686900 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 1.047897998s to process 0 records I0416 12:15:35.901442 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0416 12:15:35.901455 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-mmlsk pod in namespace openshift-dns (previous: false). I0416 12:15:36.097514 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0416 12:15:36.097534 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-58fd748686-ph2cd pod in namespace openshift-image-registry (previous: false). I0416 12:15:36.110542 1 gather_cluster_operators.go:184] Unable to get configs.samples.operator.openshift.io resource due to: configs.samples.operator.openshift.io "cluster" not found I0416 12:15:36.276102 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0416 12:15:36.276119 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-58fd748686-t9z9k pod in namespace openshift-image-registry (previous: false). I0416 12:15:36.476265 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0416 12:15:36.476282 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-5f9c7c7f49-ns7qf pod in namespace openshift-image-registry (previous: false). W0416 12:15:36.667752 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0416 12:15:36.681253 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for image-registry-5f9c7c7f49-ns7qf pod in namespace openshift-image-registry for failing operator registry (previous: false): "container \"registry\" in pod \"image-registry-5f9c7c7f49-ns7qf\" is waiting to start: ContainerCreating" I0416 12:15:36.681274 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"registry\" in pod \"image-registry-5f9c7c7f49-ns7qf\" is waiting to start: ContainerCreating" I0416 12:15:36.681288 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-crxbj pod in namespace openshift-image-registry (previous: false). I0416 12:15:36.699168 1 tasks_processing.go:74] worker 17 stopped. I0416 12:15:36.699233 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=ffc109a127cb6d56a93c357fe792cbfa80e12f25bde8ab98b0fb0fa9bf598b4c I0416 12:15:36.699284 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=94f0f52fa46ae0aae11da4197d7fbfb2b4283786957ebf77f4c49bf457dd5aac I0416 12:15:36.699314 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0416 12:15:36.699339 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=203542259cbd0492a88d2f852a9326fb0fbc585c5a5bc2f194653cf1337af7bd I0416 12:15:36.699355 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0416 12:15:36.699375 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=f3ed32e93b8e3382c5e196460c085d10b3c65c14e25daf434a27f18acfb3565a I0416 12:15:36.699428 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=bd1e85727b3dd996dbb0a308c27450643dae8622cde5779f9e237fb8a7ad7da2 I0416 12:15:36.699453 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=35d0ddcb514688a097094343ccf05aa4ca1641cfd5a9395528f899e913c4a961 I0416 12:15:36.699467 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=c8b778982d3de3d13f7fb8412d5253c85e50bad5ad42743b80c049cfdba8d788 I0416 12:15:36.699500 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=a9391d81e26a42537f05d30a59b06f85871662397dcd58387ebb63d197d28a3f I0416 12:15:36.699516 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0416 12:15:36.699540 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=c24fdb9bb403c589bc03fc5593fe1f45b222b9dd9dd4081b61771793eb55edcf I0416 12:15:36.699575 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0416 12:15:36.699605 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=9c3281f3f3fe23e3e05bd516b235ab9b88dab1ad7207784884caf0f81ebc6836 I0416 12:15:36.699618 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0416 12:15:36.699631 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=4d4996c97fd6622d645b74565ea782ab28693ec6bbc3e586eb85224de253fc88 I0416 12:15:36.699639 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0416 12:15:36.699654 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=7d6acbb64f3df4bbaf276ec0348865366f6e7667af6b7d18633b06bc4f4cd5d0 I0416 12:15:36.699790 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=1525cc38fd3fb998eccd8ea7ee9f58c102fccf03047bfbabcc69280bcd320053 I0416 12:15:36.699801 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0416 12:15:36.699811 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0416 12:15:36.699862 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0416 12:15:36.699889 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=2af9a7b8cc385ed0e6be86f275628499c88ca51621586181d9ba2c450fd6096e I0416 12:15:36.699913 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=dbe12676b46db1467c08dc79dfb5b9f5d48d785e59a38d8ea337120287c40253 I0416 12:15:36.699924 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0416 12:15:36.699946 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=a5a2f344f5ab718728ac43449b4b700602215902cd355591f91499596afbfd89 I0416 12:15:36.699961 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0416 12:15:36.699980 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=9c236379d34f08570746907db1ed285976a4735e9656021f58167695184f0fea I0416 12:15:36.699996 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=7a4a80164b627bd26ff40f57d4cebc8843d2e7d05e01fe7d4e547077ba536e93 I0416 12:15:36.700010 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=f60cd1ffdac149191762b4e9d1d20beab337ee24e164ce1a7f50fd97ee76f2f7 I0416 12:15:36.700023 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=89221849d244d7b414065dcbb8262c04e33469efdc4f0ae914fa6924a47a0ba0 I0416 12:15:36.700044 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=a52d981360bd9766bae4df304e932a4b725de83cf823669a630f8a2d72b82f4b I0416 12:15:36.700053 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0416 12:15:36.700081 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=9a348e0622f39a46d000b1de0d49ff90439914c585549ec745543448710f7f47 I0416 12:15:36.700102 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0416 12:15:36.700110 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0416 12:15:36.700117 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.064023324s to process 36 records I0416 12:15:36.881695 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0416 12:15:36.881714 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-kjqtb pod in namespace openshift-image-registry (previous: false). I0416 12:15:37.083736 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0416 12:15:37.083754 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-twlhf pod in namespace openshift-image-registry (previous: false). I0416 12:15:37.282590 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0416 12:15:37.282611 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-5d79c7df7c-4ll5r pod in namespace openshift-ingress (previous: false). I0416 12:15:37.481767 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for router-default-5d79c7df7c-4ll5r pod in namespace openshift-ingress for failing operator router (previous: false): "container \"router\" in pod \"router-default-5d79c7df7c-4ll5r\" is waiting to start: ContainerCreating" I0416 12:15:37.481787 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"router\" in pod \"router-default-5d79c7df7c-4ll5r\" is waiting to start: ContainerCreating" I0416 12:15:37.481800 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-676bd6df4c-f4jrx pod in namespace openshift-ingress (previous: false). W0416 12:15:37.667206 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0416 12:15:37.698326 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for router-default-676bd6df4c-f4jrx pod in namespace openshift-ingress for failing operator router (previous: false): "container \"router\" in pod \"router-default-676bd6df4c-f4jrx\" is waiting to start: ContainerCreating" I0416 12:15:37.698341 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"router\" in pod \"router-default-676bd6df4c-f4jrx\" is waiting to start: ContainerCreating" I0416 12:15:37.698353 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-676bd6df4c-jdzqz pod in namespace openshift-ingress (previous: false). I0416 12:15:37.878075 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0416 12:15:37.878090 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-8zzzg pod in namespace openshift-ingress-canary (previous: false). I0416 12:15:38.080770 1 tasks_processing.go:74] worker 10 stopped. I0416 12:15:38.080864 1 recorder.go:75] Recording events/openshift-dns-operator with fingerprint=ea45ca7adda142be9503b75d42318528bdd6655756ec4b5bdd8b71f360bb5353 I0416 12:15:38.080909 1 recorder.go:75] Recording events/openshift-dns with fingerprint=3f8fbb545528e11b52523fcb8707370e69cf207df24a3783f327da0b0dbd1a63 I0416 12:15:38.081006 1 recorder.go:75] Recording events/openshift-image-registry with fingerprint=dd91f74332ec67cb30e6cfb0021bdc1dca4904a2bdf2b87e65a9f108e89d997d I0416 12:15:38.081034 1 recorder.go:75] Recording events/openshift-ingress-operator with fingerprint=c1212d9362ac9d5d9a3bc2d9467c016e3b118c6976a1f043279c874f0257dee0 I0416 12:15:38.081076 1 recorder.go:75] Recording events/openshift-ingress with fingerprint=220651120485931fef8104d62e8f3bf2287a5b24eeee564eb795c46bc56da013 I0416 12:15:38.081096 1 recorder.go:75] Recording events/openshift-ingress-canary with fingerprint=e6fbc5551a48c039a1f1b23d047cbea0c5f219e639753f8733e2d2b5c71ef494 I0416 12:15:38.081103 1 recorder.go:75] Recording config/pod/openshift-ingress-canary/logs/ingress-canary-8zzzg/serve-healthcheck-canary_current.log with fingerprint=bee14510abe0b411e3e416926f6937f7339bb90755f1b5a1856e65a474152b11 I0416 12:15:38.081109 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 3.443014274s to process 7 records I0416 12:15:38.525374 1 configmapobserver.go:84] configmaps "insights-config" not found W0416 12:15:38.667188 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0416 12:15:39.666508 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0416 12:15:39.666529 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0416 12:15:39.666543 1 tasks_processing.go:74] worker 24 stopped. E0416 12:15:39.666553 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0416 12:15:39.666581 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 W0416 12:15:39.666601 1 gather.go:155] issue recording gatherer "clusterconfig" function "dvo_metrics" result "config/dvo_metrics" because of the warning: warning: the record with the same fingerprint "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" was already recorded at path "config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt", recording another one with a different path "config/dvo_metrics" I0416 12:15:39.666614 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.031132064s to process 1 records I0416 12:15:46.259732 1 configmapobserver.go:84] configmaps "insights-config" not found I0416 12:15:46.460648 1 configmapobserver.go:84] configmaps "insights-config" not found I0416 12:15:47.274328 1 tasks_processing.go:74] worker 44 stopped. I0416 12:15:47.274364 1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b I0416 12:15:47.274376 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.638474878s to process 1 records I0416 12:15:48.042206 1 tasks_processing.go:74] worker 58 stopped. I0416 12:15:48.042459 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=a07873ad19c38a900d225cd4e761acaa82a2cca4c691307b4c7c64dc85e3e73c I0416 12:15:48.042474 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.407035206s to process 1 records E0416 12:15:48.042529 1 periodic.go:254] "Unhandled Error" err="clusterconfig failed after 13.407s with: function \"machines\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"support_secret\" failed with an error, function \"machine_configs\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0416 12:15:48.043647 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "machines" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machine_healthchecks" failed with an error, function "support_secret" failed with an error, function "machine_configs" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0416 12:15:48.043662 1 periodic.go:216] Running workloads gatherer I0416 12:15:48.043678 1 tasks_processing.go:45] number of workers: 2 I0416 12:15:48.043692 1 tasks_processing.go:69] worker 1 listening for tasks. I0416 12:15:48.043696 1 tasks_processing.go:71] worker 1 working on workload_info task. I0416 12:15:48.043704 1 tasks_processing.go:69] worker 0 listening for tasks. I0416 12:15:48.043724 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0416 12:15:48.075716 1 tasks_processing.go:74] worker 0 stopped. I0416 12:15:48.075743 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 31.97556ms to process 0 records I0416 12:15:48.109893 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 22s for image data I0416 12:15:48.118112 1 gather_workloads_info.go:387] No image sha256:730d1b6988025bef0daa3a9a5d8467ec4a26b0382cc52f91c3375b4590d3518a (9ms) I0416 12:15:48.126345 1 gather_workloads_info.go:387] No image sha256:a043239802b3eb8b323d285193d2527fad0ecec98ca91d188a3472a2fac8ae04 (8ms) I0416 12:15:48.134258 1 gather_workloads_info.go:387] No image sha256:d6fbe0075cbb12bfd287c973704fadf97154c7f73e370733d976a40835e9155a (8ms) I0416 12:15:48.144308 1 gather_workloads_info.go:387] No image sha256:934d8e8c50f3c609b8eea80d1051111fe3d066fe8c65c79572072ae55fcb0a86 (10ms) I0416 12:15:48.152497 1 gather_workloads_info.go:387] No image sha256:08c5a78c8a5af04c549e2273aaf4bb452a75bf038d68aa9d01bb2aff66c30e90 (8ms) I0416 12:15:48.161345 1 gather_workloads_info.go:387] No image sha256:5ff204630794311b0b37fa7b197b933ad85d76a481bf7cdb3bcbada08f0cdcbf (9ms) I0416 12:15:48.169460 1 gather_workloads_info.go:387] No image sha256:6814f5fced0ef219d06374011c68a11a3da788a764a00a69ade435466d9ee240 (8ms) I0416 12:15:48.179977 1 gather_workloads_info.go:387] No image sha256:8f881f6f0498eaba860d711dc8e9dc066db8728f22abedb82e1dc9a8e1c30046 (11ms) I0416 12:15:48.188707 1 gather_workloads_info.go:387] No image sha256:5b6212b8f539f08e78417d8a4b7485ca0b4e7927cacd7b752742a28841bc8ccd (9ms) I0416 12:15:48.197116 1 gather_workloads_info.go:387] No image sha256:89277d8d4560d71db88c2dcc67c992a24544ca21810920b609c1d49d53b4a287 (8ms) I0416 12:15:48.218170 1 gather_workloads_info.go:387] No image sha256:e3732e356ae2324565c74cae57d5d016917314fa293ddaa3a68ee9ae030c6f07 (21ms) I0416 12:15:48.319351 1 gather_workloads_info.go:387] No image sha256:e1ba458cf6f0b3606c90880da72db8ab99cd11040bae84baebe3ff2e0d1ea075 (101ms) I0416 12:15:48.419666 1 gather_workloads_info.go:387] No image sha256:5aaea0419169e55832cc27acfe0fe3b9513a343d6bdf71d3da1575ed322245d0 (100ms) I0416 12:15:48.519110 1 gather_workloads_info.go:387] No image sha256:5c6d21c3f97366bc7ab57031cc027b67405a684bd804ce364ed5998b0685eaca (99ms) I0416 12:15:48.619086 1 gather_workloads_info.go:387] No image sha256:4b748615a43be416d52c33df03e2cdec89ededded1d6a2ade167c5a955d56e5f (100ms) I0416 12:15:48.719264 1 gather_workloads_info.go:387] No image sha256:25c148fd380b1a9db3f6039d2e0eabc489a954921452391390ba9192b2325678 (100ms) I0416 12:15:48.819085 1 gather_workloads_info.go:387] No image sha256:4733236617781e3469ffffb15e4daaa1f14ea8e1c52b426a3787a4f1f2945424 (100ms) I0416 12:15:48.918688 1 gather_workloads_info.go:387] No image sha256:673ebc8cc22c56c8f410e011b2fa950d28cf7b6420e17fdb6580d6cb10523384 (100ms) I0416 12:15:49.019572 1 gather_workloads_info.go:387] No image sha256:d3e4d4d3324f94a97ce5110eb207cf23299ed7c5f1e8d369a6583552efd87f47 (101ms) I0416 12:15:49.119327 1 gather_workloads_info.go:387] No image sha256:084aa9b0f8a6d478549dc384d4e66da13ee9b25cc98531da861cc19dee2a9e8f (100ms) I0416 12:15:49.219035 1 gather_workloads_info.go:387] No image sha256:30e597ec5d6bb96ff70a4f8688c748b659cd4fd5d73d222e8701821d236795c5 (100ms) I0416 12:15:49.318413 1 gather_workloads_info.go:387] No image sha256:55b1db6038c5beaed54c626e3343b7a8589cc0be8dc41d1a66b4deab766ff520 (99ms) I0416 12:15:49.419295 1 gather_workloads_info.go:387] No image sha256:637d41f067a5239096fc22b135181cda5113da833f1370e7a73965e83792e93a (101ms) I0416 12:15:49.519136 1 gather_workloads_info.go:387] No image sha256:3bba1358d4a0ae878ff491c0c2cbfffe60649e110b40342b878fe8fa332f8858 (100ms) I0416 12:15:49.619759 1 gather_workloads_info.go:387] No image sha256:5ac9c549d65fc1d8bc900773bebee43e9192bcec1bb5fa46afb4597230c16ac7 (101ms) I0416 12:15:49.720010 1 gather_workloads_info.go:387] No image sha256:ce138e8cf5b96557b1864ef6f27c2608bdca59be2611804366cef7169c36291e (100ms) I0416 12:15:49.818419 1 gather_workloads_info.go:387] No image sha256:000105ef5150e7079b90a613fb9e6193e2a6ef9b1908d2dce44f2395d4fd070f (98ms) I0416 12:15:49.919304 1 gather_workloads_info.go:387] No image sha256:0a4dfb8d4c1b3849319d45b4c54dff26a7238a2c08fcaa121f93073e95ab12e8 (101ms) I0416 12:15:50.019056 1 gather_workloads_info.go:387] No image sha256:48e883a3932aea9457f55cd4628d21397d429d4828f42f95d1c903d9d5395bde (100ms) I0416 12:15:50.119005 1 gather_workloads_info.go:387] No image sha256:c15ca0c0ad60fe8757c2d5d1723fcdd7a1ed6c0251a90d22a7e6cae6811d01aa (100ms) I0416 12:15:50.219129 1 gather_workloads_info.go:387] No image sha256:ab60623bb32f7e75fca71ef65137731cae347a21c7a4091dfd583fa00732721c (100ms) I0416 12:15:50.318780 1 gather_workloads_info.go:387] No image sha256:5808401268394502d335281ea1a294b07210461b986b58f91d7d1f29c0029c6d (100ms) I0416 12:15:50.419841 1 gather_workloads_info.go:387] No image sha256:d64bea34bf3e1bb0b3a701c3ff14e66665afc1b050f28124ad7e6888eaec3a81 (101ms) I0416 12:15:50.519875 1 gather_workloads_info.go:387] No image sha256:875c77e5d144f03fb91d8cee0259f6966683ca88d1bf818dbf4652c16b70312c (100ms) I0416 12:15:50.618638 1 gather_workloads_info.go:387] No image sha256:15677f0b70e6aa2dfaf088e45fc1a425c22bf6fda326b8116f87e88b6694dfab (99ms) I0416 12:15:50.618667 1 tasks_processing.go:74] worker 1 stopped. E0416 12:15:50.618678 1 gather.go:140] gatherer "workloads" function "workload_info" failed with the error: no running pods found for the insights-runtime-extractor statefulset I0416 12:15:50.618970 1 recorder.go:75] Recording config/workload_info with fingerprint=1d59e0dd0a11e498b40f462b93d0af4ef9e03ee117403f0c57319de3e8948386 I0416 12:15:50.618988 1 gather.go:177] gatherer "workloads" function "workload_info" took 2.574963237s to process 1 records E0416 12:15:50.619017 1 periodic.go:254] "Unhandled Error" err="workloads failed after 2.575s with: function \"workload_info\" failed with an error" I0416 12:15:50.620127 1 controllerstatus.go:89] name=periodic-workloads healthy=false reason=PeriodicGatherFailed message=Source workloads could not be retrieved: function "workload_info" failed with an error I0416 12:15:50.620142 1 periodic.go:216] Running conditional gatherer I0416 12:15:50.625644 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.21.9/gathering_rules I0416 12:15:50.633038 1 conditional_gatherer.go:107] Get "https://console.redhat.com/api/gathering/v2/4.21.9/gathering_rules": dial tcp: lookup console.redhat.com on 172.30.0.10:53: read udp 10.128.0.20:51508->172.30.0.10:53: read: connection refused E0416 12:15:50.633282 1 conditional_gatherer.go:322] unable to update alerts cache: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0416 12:15:50.633344 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0416 12:15:50.638940 1 conditional_gatherer.go:392] cluster version is '4.21.9' E0416 12:15:50.638953 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0416 12:15:50.638959 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0416 12:15:50.638963 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0416 12:15:50.638966 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0416 12:15:50.638970 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0416 12:15:50.638973 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0416 12:15:50.638976 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0416 12:15:50.638979 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0416 12:15:50.638982 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0416 12:15:50.638997 1 tasks_processing.go:45] number of workers: 3 I0416 12:15:50.639010 1 tasks_processing.go:69] worker 2 listening for tasks. I0416 12:15:50.639015 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0416 12:15:50.639021 1 tasks_processing.go:69] worker 0 listening for tasks. I0416 12:15:50.639032 1 tasks_processing.go:69] worker 1 listening for tasks. I0416 12:15:50.639047 1 tasks_processing.go:74] worker 1 stopped. I0416 12:15:50.639059 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0416 12:15:50.639071 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0416 12:15:50.639106 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=7034af97e7e41c22e4b775abdd4b9066c8ebb19da33eb7f69f39bfd2eb5f6406 I0416 12:15:50.639120 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.09µs to process 1 records I0416 12:15:50.639151 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=0394430c431eec4d48bb1811a90918e95161d2282c59af26f2473613cc0959db I0416 12:15:50.639160 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 907ns to process 1 records I0416 12:15:50.639166 1 tasks_processing.go:74] worker 0 stopped. I0416 12:15:50.639355 1 tasks_processing.go:74] worker 2 stopped. I0416 12:15:50.639369 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 264.23µs to process 0 records I0416 12:15:50.639399 1 controllerstatus.go:89] name=periodic-conditional healthy=false reason=NotAvailable message=Get "https://console.redhat.com/api/gathering/v2/4.21.9/gathering_rules": dial tcp: lookup console.redhat.com on 172.30.0.10:53: read udp 10.128.0.20:51508->172.30.0.10:53: read: connection refused I0416 12:15:50.639418 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=359de9c990c741675cec72fda96b5c3682221efdb4799f5eaa6e9805bcd3b5c1 I0416 12:15:50.662983 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=98c58b4714315f5d73283fcb752ed3320491b166e40e2ae0fd7e2e06ad4f7e27 I0416 12:15:50.663099 1 diskrecorder.go:70] Writing 101 records to /var/lib/insights-operator/insights-2026-04-16-121550.tar.gz I0416 12:15:50.668846 1 diskrecorder.go:51] Wrote 101 records to disk in 5ms I0416 12:15:50.668883 1 periodic.go:285] Gathering cluster info every 2h0m0s I0416 12:15:50.668899 1 periodic.go:286] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0416 12:16:03.861233 1 configmapobserver.go:84] configmaps "insights-config" not found I0416 12:16:49.431348 1 observer_polling.go:111] Observed file "/var/run/secrets/serving-cert/tls.key" has been created (hash="17d0488b0408526c85601c16cc49280048da2a22bc94292463f2fc4f9db8051c") W0416 12:16:49.431380 1 builder.go:160] Restart triggered because of file /var/run/secrets/serving-cert/tls.key was created