I0428 11:10:10.169960 1 cmd.go:253] Using service-serving-cert provided certificates I0428 11:10:10.170398 1 observer_polling.go:159] Starting file observer I0428 11:10:10.582779 1 operator.go:76] Starting insights-operator v0.0.0-master+$Format:%H$ I0428 11:10:10.582966 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0428 11:10:10.583485 1 secure_serving.go:57] Forcing use of http/1.1 only W0428 11:10:10.583506 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0428 11:10:10.583510 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0428 11:10:10.583514 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0428 11:10:10.583517 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0428 11:10:10.583519 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0428 11:10:10.583522 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0428 11:10:10.583732 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0428 11:10:10.588214 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"9ee9d292-3c45-403f-947c-a3a161169b7d", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "HyperShiftOnlyDynamicResourceAllocation", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryption", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLM", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "OpenShiftPodSecurityAdmission", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}} I0428 11:10:10.588198 1 operator.go:141] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNSInstall GatewayAPI GatewayAPIController HighlyAvailableArbiter HyperShiftOnlyDynamicResourceAllocation ImageStreamImportMode ImageVolume KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS ManagedBootImagesAzure ManagedBootImagesvSphere MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages PreconfiguredUDNAddresses ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SigstoreImageVerification SigstoreImageVerificationPKI StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSDualStackInstall AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureDualStackInstall AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement CBORServingAndStorage CRDCompatibilityRequirementOperator ClientsAllowCBOR ClientsPreferCBOR ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterAPIMachineManagement ClusterAPIMachineManagementVSphere ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud EtcdBackendQuota EventTTL EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall GCPDualStackInstall ImageModeStatusReporting IngressControllerDynamicConfigurationManager InsightsConfig InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryption KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesCPMS MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutableCSINodeAllocatableCount MutatingAdmissionPolicy NewOLM NewOLMBoxCutterRuntime NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterInstall NutanixMultiSubnets OSStreams OVNObservability OnPremDNSRecords OpenShiftPodSecurityAdmission ProvisioningRequestAvailable SELinuxMount ShortCertRotation SignatureStores TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeGroupSnapshot] I0428 11:10:10.588647 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0428 11:10:10.588678 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0428 11:10:10.588689 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0428 11:10:10.588723 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0428 11:10:10.588733 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0428 11:10:10.588734 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0428 11:10:10.588952 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0428 11:10:10.589202 1 secure_serving.go:213] Serving securely on [::]:8443 I0428 11:10:10.589228 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0428 11:10:10.600664 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0428 11:10:10.600691 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0428 11:10:10.600791 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0428 11:10:10.615327 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0428 11:10:10.615355 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0428 11:10:10.621757 1 secretconfigobserver.go:119] support secret does not exist I0428 11:10:10.631003 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0428 11:10:10.636209 1 secretconfigobserver.go:119] support secret does not exist I0428 11:10:10.642054 1 recorder.go:176] Pruning old reports every 6h51m42s, max age is 288h0m0s I0428 11:10:10.647535 1 periodic.go:216] Running clusterconfig gatherer I0428 11:10:10.647541 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0428 11:10:10.647543 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0428 11:10:10.647552 1 insightsreport.go:296] Starting report retriever I0428 11:10:10.647559 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0428 11:10:10.647563 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0428 11:10:10.647576 1 tasks_processing.go:45] number of workers: 64 I0428 11:10:10.647589 1 tasks_processing.go:69] worker 0 listening for tasks. I0428 11:10:10.647596 1 tasks_processing.go:69] worker 6 listening for tasks. I0428 11:10:10.647602 1 tasks_processing.go:69] worker 1 listening for tasks. I0428 11:10:10.647607 1 tasks_processing.go:69] worker 2 listening for tasks. I0428 11:10:10.647613 1 tasks_processing.go:69] worker 3 listening for tasks. I0428 11:10:10.647618 1 tasks_processing.go:69] worker 4 listening for tasks. I0428 11:10:10.647624 1 tasks_processing.go:71] worker 3 working on mutating_webhook_configurations task. I0428 11:10:10.647630 1 tasks_processing.go:69] worker 5 listening for tasks. I0428 11:10:10.647636 1 tasks_processing.go:71] worker 5 working on storage_classes task. I0428 11:10:10.647657 1 tasks_processing.go:69] worker 37 listening for tasks. I0428 11:10:10.647672 1 tasks_processing.go:69] worker 7 listening for tasks. I0428 11:10:10.647670 1 tasks_processing.go:69] worker 52 listening for tasks. I0428 11:10:10.647680 1 tasks_processing.go:69] worker 38 listening for tasks. I0428 11:10:10.647684 1 tasks_processing.go:69] worker 23 listening for tasks. I0428 11:10:10.647689 1 tasks_processing.go:69] worker 39 listening for tasks. I0428 11:10:10.647696 1 tasks_processing.go:69] worker 24 listening for tasks. I0428 11:10:10.647705 1 tasks_processing.go:69] worker 31 listening for tasks. I0428 11:10:10.647816 1 tasks_processing.go:69] worker 25 listening for tasks. I0428 11:10:10.647842 1 tasks_processing.go:69] worker 56 listening for tasks. I0428 11:10:10.647849 1 tasks_processing.go:69] worker 14 listening for tasks. I0428 11:10:10.647859 1 tasks_processing.go:69] worker 26 listening for tasks. I0428 11:10:10.647841 1 tasks_processing.go:69] worker 27 listening for tasks. I0428 11:10:10.647868 1 tasks_processing.go:69] worker 53 listening for tasks. I0428 11:10:10.647870 1 tasks_processing.go:69] worker 11 listening for tasks. I0428 11:10:10.647877 1 tasks_processing.go:69] worker 54 listening for tasks. I0428 11:10:10.647878 1 tasks_processing.go:69] worker 20 listening for tasks. I0428 11:10:10.647882 1 tasks_processing.go:69] worker 12 listening for tasks. I0428 11:10:10.647852 1 tasks_processing.go:69] worker 22 listening for tasks. I0428 11:10:10.647888 1 tasks_processing.go:69] worker 19 listening for tasks. I0428 11:10:10.647887 1 tasks_processing.go:69] worker 55 listening for tasks. I0428 11:10:10.647891 1 tasks_processing.go:69] worker 21 listening for tasks. I0428 11:10:10.647894 1 tasks_processing.go:69] worker 13 listening for tasks. I0428 11:10:10.647899 1 tasks_processing.go:69] worker 49 listening for tasks. I0428 11:10:10.647900 1 tasks_processing.go:69] worker 50 listening for tasks. I0428 11:10:10.647894 1 tasks_processing.go:69] worker 46 listening for tasks. I0428 11:10:10.647905 1 tasks_processing.go:69] worker 28 listening for tasks. I0428 11:10:10.647907 1 tasks_processing.go:69] worker 29 listening for tasks. I0428 11:10:10.647899 1 tasks_processing.go:69] worker 57 listening for tasks. I0428 11:10:10.647918 1 tasks_processing.go:69] worker 41 listening for tasks. I0428 11:10:10.647918 1 tasks_processing.go:69] worker 33 listening for tasks. I0428 11:10:10.647920 1 tasks_processing.go:69] worker 35 listening for tasks. I0428 11:10:10.647906 1 tasks_processing.go:69] worker 59 listening for tasks. I0428 11:10:10.647922 1 tasks_processing.go:69] worker 32 listening for tasks. I0428 11:10:10.647925 1 tasks_processing.go:69] worker 15 listening for tasks. I0428 11:10:10.647929 1 tasks_processing.go:69] worker 36 listening for tasks. I0428 11:10:10.647935 1 tasks_processing.go:69] worker 61 listening for tasks. I0428 11:10:10.647938 1 tasks_processing.go:69] worker 9 listening for tasks. I0428 11:10:10.647939 1 tasks_processing.go:69] worker 44 listening for tasks. I0428 11:10:10.647937 1 tasks_processing.go:71] worker 6 working on openstack_dataplanedeployments task. I0428 11:10:10.647945 1 tasks_processing.go:69] worker 62 listening for tasks. I0428 11:10:10.647944 1 tasks_processing.go:69] worker 45 listening for tasks. I0428 11:10:10.647912 1 tasks_processing.go:69] worker 34 listening for tasks. I0428 11:10:10.647915 1 tasks_processing.go:69] worker 10 listening for tasks. I0428 11:10:10.647961 1 tasks_processing.go:69] worker 63 listening for tasks. I0428 11:10:10.647953 1 tasks_processing.go:69] worker 60 listening for tasks. I0428 11:10:10.647965 1 tasks_processing.go:69] worker 40 listening for tasks. I0428 11:10:10.647919 1 tasks_processing.go:69] worker 16 listening for tasks. I0428 11:10:10.647914 1 tasks_processing.go:69] worker 48 listening for tasks. I0428 11:10:10.647883 1 tasks_processing.go:69] worker 58 listening for tasks. I0428 11:10:10.647928 1 tasks_processing.go:69] worker 42 listening for tasks. I0428 11:10:10.647929 1 tasks_processing.go:71] worker 0 working on support_secret task. I0428 11:10:10.647988 1 tasks_processing.go:69] worker 18 listening for tasks. I0428 11:10:10.647624 1 tasks_processing.go:71] worker 4 working on pdbs task. I0428 11:10:10.647927 1 tasks_processing.go:69] worker 8 listening for tasks. I0428 11:10:10.647928 1 tasks_processing.go:69] worker 17 listening for tasks. I0428 11:10:10.648033 1 tasks_processing.go:71] worker 26 working on silenced_alerts task. I0428 11:10:10.648037 1 tasks_processing.go:71] worker 55 working on authentication task. I0428 11:10:10.648041 1 tasks_processing.go:71] worker 39 working on cost_management_metrics_configs task. I0428 11:10:10.648047 1 tasks_processing.go:71] worker 17 working on operators_pods_and_events task. I0428 11:10:10.648054 1 tasks_processing.go:71] worker 13 working on ceph_cluster task. I0428 11:10:10.648098 1 tasks_processing.go:71] worker 54 working on nodenetworkstates task. I0428 11:10:10.648149 1 tasks_processing.go:71] worker 11 working on infrastructures task. I0428 11:10:10.648048 1 tasks_processing.go:71] worker 21 working on metrics task. I0428 11:10:10.648043 1 tasks_processing.go:71] worker 8 working on image task. I0428 11:10:10.648322 1 tasks_processing.go:69] worker 30 listening for tasks. I0428 11:10:10.648380 1 tasks_processing.go:71] worker 30 working on openshift_machine_api_events task. I0428 11:10:10.647935 1 tasks_processing.go:69] worker 43 listening for tasks. I0428 11:10:10.647938 1 tasks_processing.go:71] worker 2 working on schedulers task. I0428 11:10:10.647941 1 tasks_processing.go:69] worker 51 listening for tasks. I0428 11:10:10.648454 1 tasks_processing.go:71] worker 43 working on ingress_certificates task. I0428 11:10:10.648110 1 tasks_processing.go:71] worker 20 working on storage_cluster task. I0428 11:10:10.648113 1 tasks_processing.go:71] worker 61 working on image_pruners task. I0428 11:10:10.648114 1 tasks_processing.go:71] worker 37 working on machines task. I0428 11:10:10.648115 1 tasks_processing.go:71] worker 12 working on nodenetworkconfigurationpolicies task. I0428 11:10:10.648118 1 tasks_processing.go:71] worker 49 working on sap_datahubs task. I0428 11:10:10.647907 1 tasks_processing.go:69] worker 47 listening for tasks. I0428 11:10:10.648702 1 tasks_processing.go:71] worker 47 working on service_accounts task. I0428 11:10:10.648119 1 tasks_processing.go:71] worker 22 working on olm_operators task. I0428 11:10:10.648119 1 tasks_processing.go:71] worker 7 working on machine_healthchecks task. I0428 11:10:10.648123 1 tasks_processing.go:71] worker 50 working on version task. I0428 11:10:10.648127 1 tasks_processing.go:71] worker 25 working on machine_sets task. I0428 11:10:10.648129 1 tasks_processing.go:71] worker 52 working on openstack_controlplanes task. I0428 11:10:10.648125 1 tasks_processing.go:71] worker 19 working on tsdb_status task. I0428 11:10:10.648130 1 tasks_processing.go:71] worker 46 working on node_features task. I0428 11:10:10.648133 1 tasks_processing.go:71] worker 38 working on lokistack task. I0428 11:10:10.648132 1 tasks_processing.go:71] worker 24 working on monitoring_persistent_volumes task. I0428 11:10:10.648136 1 tasks_processing.go:71] worker 28 working on nodes task. I0428 11:10:10.648131 1 tasks_processing.go:71] worker 27 working on image_registries task. I0428 11:10:10.648139 1 tasks_processing.go:71] worker 29 working on sap_config task. I0428 11:10:10.648142 1 tasks_processing.go:71] worker 57 working on install_plans task. I0428 11:10:10.648136 1 tasks_processing.go:71] worker 56 working on feature_gates task. I0428 11:10:10.648144 1 tasks_processing.go:71] worker 53 working on crds task. I0428 11:10:10.648139 1 tasks_processing.go:71] worker 31 working on validating_webhook_configurations task. I0428 11:10:10.648145 1 tasks_processing.go:71] worker 41 working on proxies task. I0428 11:10:10.648138 1 tasks_processing.go:71] worker 23 working on active_alerts task. I0428 11:10:10.648152 1 tasks_processing.go:71] worker 63 working on machine_configs task. I0428 11:10:10.648151 1 tasks_processing.go:71] worker 33 working on sap_pods task. I0428 11:10:10.648155 1 tasks_processing.go:71] worker 32 working on overlapping_namespace_uids task. I0428 11:10:10.648156 1 tasks_processing.go:71] worker 9 working on dvo_metrics task. I0428 11:10:10.648157 1 tasks_processing.go:71] worker 15 working on certificate_signing_requests task. I0428 11:10:10.648150 1 tasks_processing.go:71] worker 14 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0428 11:10:10.648162 1 tasks_processing.go:71] worker 44 working on openshift_logging task. I0428 11:10:10.648165 1 tasks_processing.go:71] worker 62 working on config_maps task. I0428 11:10:10.648160 1 tasks_processing.go:71] worker 36 working on openstack_dataplanenodesets task. I0428 11:10:10.648161 1 tasks_processing.go:71] worker 59 working on oauths task. I0428 11:10:10.648169 1 tasks_processing.go:71] worker 45 working on container_runtime_configs task. I0428 11:10:10.648179 1 tasks_processing.go:71] worker 48 working on node_logs task. I0428 11:10:10.648171 1 tasks_processing.go:71] worker 35 working on container_images task. I0428 11:10:10.648171 1 tasks_processing.go:71] worker 34 working on ingress task. I0428 11:10:10.648173 1 tasks_processing.go:71] worker 10 working on operators task. I0428 11:10:10.648184 1 tasks_processing.go:71] worker 60 working on pod_network_connectivity_checks task. I0428 11:10:10.648188 1 tasks_processing.go:71] worker 40 working on openstack_version task. I0428 11:10:10.648184 1 tasks_processing.go:71] worker 42 working on cluster_apiserver task. I0428 11:10:10.648184 1 tasks_processing.go:71] worker 18 working on qemu_kubevirt_launcher_logs task. I0428 11:10:10.647930 1 tasks_processing.go:71] worker 1 working on jaegers task. I0428 11:10:10.648185 1 tasks_processing.go:71] worker 58 working on machine_config_pools task. I0428 11:10:10.648191 1 tasks_processing.go:71] worker 16 working on aggregated_monitoring_cr_names task. I0428 11:10:10.648465 1 tasks_processing.go:71] worker 51 working on clusterroles task. I0428 11:10:10.653902 1 tasks_processing.go:71] worker 54 working on networks task. I0428 11:10:10.653909 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 5.736705ms to process 0 records E0428 11:10:10.654259 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.249.241:9091: connect: connection refused I0428 11:10:10.654270 1 tasks_processing.go:71] worker 6 working on machine_autoscalers task. I0428 11:10:10.654279 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 6.317919ms to process 0 records E0428 11:10:10.654276 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.249.241:9091: connect: connection refused E0428 11:10:10.654289 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.249.241:9091: connect: connection refused I0428 11:10:10.654302 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 5.001191ms to process 0 records E0428 11:10:10.654310 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.249.241:9091: connect: connection refused I0428 11:10:10.654325 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 6.012227ms to process 0 records I0428 11:10:10.654378 1 tasks_processing.go:74] worker 21 stopped. I0428 11:10:10.654381 1 tasks_processing.go:74] worker 19 stopped. E0428 11:10:10.654445 1 gather_active_alerts.go:64] Unable to retrieve most recent alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp 172.30.151.82:9094: connect: connection refused I0428 11:10:10.654465 1 tasks_processing.go:74] worker 23 stopped. E0428 11:10:10.654478 1 gather.go:140] gatherer "clusterconfig" function "active_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=true": dial tcp 172.30.151.82:9094: connect: connection refused I0428 11:10:10.654497 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 4.082599ms to process 0 records E0428 11:10:10.654513 1 gather_silenced_alerts.go:52] Unable to retrieve silenced alerts: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp 172.30.151.82:9094: connect: connection refused I0428 11:10:10.654529 1 tasks_processing.go:74] worker 26 stopped. E0428 11:10:10.654536 1 gather.go:140] gatherer "clusterconfig" function "silenced_alerts" failed with the error: Get "https://alertmanager-main.openshift-monitoring.svc:9094/api/v2/alerts?active=false&inhibited=false&silenced=true": dial tcp 172.30.151.82:9094: connect: connection refused I0428 11:10:10.654578 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 6.487646ms to process 0 records I0428 11:10:10.655797 1 tasks_processing.go:74] worker 39 stopped. I0428 11:10:10.655811 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 7.742691ms to process 0 records I0428 11:10:10.671546 1 tasks_processing.go:74] worker 0 stopped. E0428 11:10:10.671564 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0428 11:10:10.671573 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 23.546047ms to process 0 records I0428 11:10:10.671623 1 tasks_processing.go:74] worker 45 stopped. I0428 11:10:10.671628 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 18.92771ms to process 0 records I0428 11:10:10.671769 1 tasks_processing.go:74] worker 60 stopped. E0428 11:10:10.671783 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0428 11:10:10.671817 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 18.774114ms to process 0 records I0428 11:10:10.671911 1 tasks_processing.go:74] worker 8 stopped. I0428 11:10:10.671967 1 recorder.go:75] Recording config/image with fingerprint=cabf50451544588daac60400cc9e6ae917d242fb26a285e0c7b50b17f2c58602 I0428 11:10:10.671974 1 controller.go:129] Initializing last reported time to 0001-01-01T00:00:00Z I0428 11:10:10.671993 1 controller.go:368] The initial operator extension status is healthy I0428 11:10:10.671993 1 gather.go:177] gatherer "clusterconfig" function "image" took 23.460524ms to process 1 records I0428 11:10:10.672007 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0428 11:10:10.672014 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0428 11:10:10.672017 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0428 11:10:10.672033 1 controller.go:531] The operator is still being initialized I0428 11:10:10.672042 1 controller.go:554] The operator is healthy I0428 11:10:10.672102 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=ec37146014278e1b99634c29ec3cffcc982d3552667f56ee6c2c44adfb9f2d07 I0428 11:10:10.672105 1 tasks_processing.go:74] worker 5 stopped. I0428 11:10:10.672124 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=9c39a3cbcd32dc587f2693fc03b3eb35ef4532e0968d292f56db251b2358db42 I0428 11:10:10.672135 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 24.126346ms to process 2 records I0428 11:10:10.672147 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 19.435576ms to process 0 records I0428 11:10:10.672160 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0428 11:10:10.672207 1 tasks_processing.go:74] worker 36 stopped. I0428 11:10:10.672219 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s I0428 11:10:10.672312 1 tasks_processing.go:74] worker 11 stopped. W0428 11:10:10.672503 1 operator.go:328] started I0428 11:10:10.672531 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0428 11:10:10.672692 1 recorder.go:75] Recording config/infrastructure with fingerprint=68ae95d3620fbb9a63e20420e9be478ab9aaf5b11490e89ba694e6cfe5f77e60 I0428 11:10:10.672704 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 23.81978ms to process 1 records I0428 11:10:10.672741 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 19.133165ms to process 0 records I0428 11:10:10.672765 1 tasks_processing.go:74] worker 40 stopped. I0428 11:10:10.672844 1 tasks_processing.go:74] worker 55 stopped. I0428 11:10:10.672887 1 recorder.go:75] Recording config/authentication with fingerprint=8144cc5d7a0a07e518b13cae67c275fd2f5cf250c7939a60e21f67139783f144 I0428 11:10:10.672896 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 24.1574ms to process 1 records I0428 11:10:10.672973 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=f8fd9ae597ad20c1e263426d35a4a4816406c7028142955d82fe401af316efd8 I0428 11:10:10.672975 1 tasks_processing.go:74] worker 4 stopped. I0428 11:10:10.672989 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=0e1ba4807e88e11f3fb4b01cee56820ea9d8e435cf108da6a53be63e91129eec I0428 11:10:10.673001 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=b4e94b997aabcdab295630a302d2a78909a2fb93d897085bdb33992f74b9e71e I0428 11:10:10.673016 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=0e603fe9b9c93da098a988cbb4c3863335b8cf005cc604c6e718d0339b6c46b5 I0428 11:10:10.673034 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=7e3c3f92593d6933aa2489fa57322b9572dc44548fb6c373d1d27979eae6d7fb I0428 11:10:10.673048 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=41ede571694a12c7d78d25b215326072258d348bed892f3790c54192b2c4d52c I0428 11:10:10.673062 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=37f91e4bcc4287d5d6fdf4e62373af20a66b5f0baad2e24c17ff403aa25fe623 I0428 11:10:10.673079 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=ffeaa8df0a5b6f3a5edb360f836bda040922b401196fad9f3388dea7725cfc32 I0428 11:10:10.673094 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=9d02e428e6901630ecff2ed5d96f510466ad603ac2a76fc5b26fb79f5164a1d0 I0428 11:10:10.673109 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=6d100161aa3b73cb1f742f3b8b56e4003397e8b2c0c22c9fa6ea79fbc10b9296 I0428 11:10:10.673125 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=e3cd0834876ec0a1fbc0ee9c970be274300af56d762385d22c1c16b0030a62fd I0428 11:10:10.673133 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 24.180292ms to process 11 records I0428 11:10:10.673208 1 tasks_processing.go:74] worker 3 stopped. I0428 11:10:10.673267 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=8783b1e6630fce2a899e4316de2464921624156308c0307237dfdf4cec640832 I0428 11:10:10.673293 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=31b7c3fa85851d4d4b46c98f1f34d0ddddd0236a544d358835f509fbaa52a938 I0428 11:10:10.673317 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=8fe93c8ef2882465e60a003251ca0e7f7b557db2509766b7635e2565e0f4bf87 I0428 11:10:10.673323 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 24.620329ms to process 3 records I0428 11:10:10.673398 1 tasks_processing.go:74] worker 61 stopped. I0428 11:10:10.673509 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=2fb5f916188db9357b290df60ddcc5bfc18762615b5e66a71cca85d14a8eef91 I0428 11:10:10.673519 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 23.827244ms to process 1 records I0428 11:10:10.673597 1 tasks_processing.go:74] worker 34 stopped. I0428 11:10:10.673629 1 recorder.go:75] Recording config/ingress with fingerprint=f4d8afdb9ca489c36a818b3486b5bf381846a49681a405aacf728b6b9e552bc7 I0428 11:10:10.673637 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 19.447017ms to process 1 records I0428 11:10:10.673727 1 tasks_processing.go:74] worker 59 stopped. I0428 11:10:10.673805 1 recorder.go:75] Recording config/oauth with fingerprint=072c6ff50f135f34e31573780ef2dfb8d62ebca56248c3f31deda2c8a6e6c4ff I0428 11:10:10.673813 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 19.816014ms to process 1 records I0428 11:10:10.677777 1 tasks_processing.go:74] worker 6 stopped. I0428 11:10:10.677793 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 23.496792ms to process 0 records I0428 11:10:10.677805 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 24.781852ms to process 0 records I0428 11:10:10.677810 1 tasks_processing.go:74] worker 58 stopped. I0428 11:10:10.677935 1 tasks_processing.go:74] worker 1 stopped. I0428 11:10:10.677959 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 24.926955ms to process 0 records I0428 11:10:10.689126 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0428 11:10:10.689143 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0428 11:10:10.689172 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0428 11:10:10.691702 1 tasks_processing.go:74] worker 7 stopped. E0428 11:10:10.691730 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0428 11:10:10.691744 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 42.784205ms to process 0 records I0428 11:10:10.693362 1 tasks_processing.go:74] worker 38 stopped. I0428 11:10:10.693378 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 43.949699ms to process 0 records I0428 11:10:10.693386 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 44.886915ms to process 0 records I0428 11:10:10.693390 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 44.782209ms to process 0 records I0428 11:10:10.693394 1 tasks_processing.go:74] worker 12 stopped. I0428 11:10:10.693398 1 tasks_processing.go:74] worker 25 stopped. I0428 11:10:10.693411 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 44.396569ms to process 0 records I0428 11:10:10.693415 1 tasks_processing.go:74] worker 20 stopped. I0428 11:10:10.693422 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 44.783887ms to process 0 records E0428 11:10:10.693428 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0428 11:10:10.693434 1 gather.go:177] gatherer "clusterconfig" function "machines" took 44.879057ms to process 0 records I0428 11:10:10.693438 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 43.632824ms to process 0 records I0428 11:10:10.693443 1 tasks_processing.go:74] worker 29 stopped. I0428 11:10:10.693447 1 tasks_processing.go:74] worker 49 stopped. I0428 11:10:10.693450 1 tasks_processing.go:74] worker 37 stopped. I0428 11:10:10.693463 1 tasks_processing.go:74] worker 52 stopped. I0428 11:10:10.693479 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 44.311805ms to process 0 records I0428 11:10:10.693496 1 tasks_processing.go:74] worker 13 stopped. I0428 11:10:10.693504 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 45.434832ms to process 0 records E0428 11:10:10.694409 1 gather_node_features.go:86] GatherNodeFeatures: NodeFeatures resource not found in openshift-nfd namespace (may not be installed) I0428 11:10:10.694424 1 tasks_processing.go:74] worker 46 stopped. I0428 11:10:10.694434 1 gather.go:177] gatherer "clusterconfig" function "node_features" took 45.058968ms to process 0 records I0428 11:10:10.694499 1 gather_logs.go:145] no pods in namespace were found I0428 11:10:10.694510 1 tasks_processing.go:74] worker 24 stopped. I0428 11:10:10.694515 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 44.960646ms to process 0 records I0428 11:10:10.694519 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 41.514781ms to process 0 records I0428 11:10:10.694524 1 tasks_processing.go:74] worker 18 stopped. I0428 11:10:10.694629 1 tasks_processing.go:74] worker 2 stopped. I0428 11:10:10.694759 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=c9933c894ce996618c3783551c2f9f4325db975f86962cd70737dc14808009fa I0428 11:10:10.694781 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 46.183537ms to process 1 records I0428 11:10:10.694871 1 tasks_processing.go:74] worker 54 stopped. I0428 11:10:10.695419 1 recorder.go:75] Recording config/network with fingerprint=3761961533b66e8f73c1bfbe436cbdc63a38bdbdecbf76faef0f17b746f5f4fc I0428 11:10:10.695443 1 gather.go:177] gatherer "clusterconfig" function "networks" took 40.797342ms to process 1 records I0428 11:10:10.696659 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 44.54825ms to process 0 records I0428 11:10:10.696600 1 tasks_processing.go:74] worker 44 stopped. W0428 11:10:10.697892 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0428 11:10:10.698379 1 tasks_processing.go:74] worker 48 stopped. I0428 11:10:10.698391 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 45.606397ms to process 0 records I0428 11:10:10.698666 1 tasks_processing.go:74] worker 28 stopped. I0428 11:10:10.699020 1 recorder.go:75] Recording config/node/ip-10-0-0-109.ec2.internal with fingerprint=0dcf849493588241a6143068033e14d86f51aec4e3b813c4215a78e314f53b23 I0428 11:10:10.699080 1 recorder.go:75] Recording config/node/ip-10-0-1-135.ec2.internal with fingerprint=ed8d51720542d2516f582e5d70b3f8d151e69fe40ec7fa7d795a9a4f75438f9a I0428 11:10:10.699134 1 recorder.go:75] Recording config/node/ip-10-0-2-140.ec2.internal with fingerprint=c82548159b9a44351bce0a8e8e4736edfde2c816b67ab47121d9c55ffef5e017 I0428 11:10:10.699143 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 49.078137ms to process 3 records I0428 11:10:10.699191 1 recorder.go:75] Recording config/proxy with fingerprint=0f089260552846c5cc5184c1000e57f56986bf7833d2e52e178370cce0ee2666 I0428 11:10:10.699199 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 48.580227ms to process 1 records I0428 11:10:10.699223 1 tasks_processing.go:74] worker 41 stopped. I0428 11:10:10.699284 1 tasks_processing.go:74] worker 27 stopped. I0428 11:10:10.699574 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=25425d953597cc1ffdabe58857f775961c0d21bea7eedbce23c9d86957dcc0c2 I0428 11:10:10.699596 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 49.55452ms to process 1 records I0428 11:10:10.701097 1 base_controller.go:82] Caches are synced for ConfigController I0428 11:10:10.701112 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0428 11:10:10.702064 1 tasks_processing.go:74] worker 30 stopped. I0428 11:10:10.702085 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 53.641434ms to process 0 records I0428 11:10:10.702096 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 51.431269ms to process 0 records I0428 11:10:10.702104 1 tasks_processing.go:74] worker 33 stopped. I0428 11:10:10.702207 1 tasks_processing.go:74] worker 56 stopped. I0428 11:10:10.702390 1 recorder.go:75] Recording config/featuregate with fingerprint=299f3b7830f96ec46e1c6eb15b7b14b85076645a8968139899d19ff8de4736f7 I0428 11:10:10.702414 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 52.132939ms to process 1 records I0428 11:10:10.702489 1 tasks_processing.go:74] worker 31 stopped. I0428 11:10:10.702548 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=82fa9454252e41c0c27674c0d20ae8081d773c562fc5208dce21a7c996dc0465 I0428 11:10:10.702640 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=fcb944f402e8ff0e7b9fce91a78afbff8cbe4869798d4070ea3c43642b32bb52 I0428 11:10:10.702815 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=a52010a640b63a3fd0bdf24f87a22b6d88e17d1e77bf1ebac4f62dab9815b2bb I0428 11:10:10.702869 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=2b9aff4e2ad05ab2781900dcd0f8d73032300c5ba1effcc3fa13cc471acb255f I0428 11:10:10.702912 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=4327c641f034883e972156d0c740ccfdb86c0c12c41f45efaac94fa1dcca5c91 I0428 11:10:10.702956 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=7da41207d0cf6aa8981c6d843a6ea27baacb03c05f36066b37062c8ba6d29a55 I0428 11:10:10.703001 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=ad655817403ebd2922270f0a664ccb76b03a09b807d4368c91b9ad774f160f6d I0428 11:10:10.703050 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=9b2a063e7740ccc408dd6aadf1d42d923cdb09e7a528109a70bef220bc30e867 I0428 11:10:10.703100 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=84c5760e644d79d96bad15e47184725c4635d0074ae1e90fe4fb02111672e0b2 I0428 11:10:10.703199 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=8e2e4cedf540fb5e4aaa967f594abb0e3060cdb61f76a52d1d57593383853f46 I0428 11:10:10.703244 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=0883e24752c00c7dece4dac71a24766ad7a1fd6b027045fc96b5c72ae96dc4d7 I0428 11:10:10.703321 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=7fd01f29d6e2afca61b04e0cf764b97b4bf06b3838bfa99a624c7b09493cd455 I0428 11:10:10.703370 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=8bfb0fa42be6aa4eeb3d53f39afd841638514026291c8cc7764ee2c231e1bbaa I0428 11:10:10.703383 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 52.041107ms to process 13 records I0428 11:10:10.707543 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 55.916694ms to process 0 records I0428 11:10:10.707438 1 tasks_processing.go:74] worker 15 stopped. I0428 11:10:10.708297 1 tasks_processing.go:74] worker 42 stopped. I0428 11:10:10.708802 1 recorder.go:75] Recording config/apiserver with fingerprint=59e322c09940ec091f7f93ad9234d2c046049fb38b7b8ae447f8b6c16ca46510 I0428 11:10:10.708834 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 55.244095ms to process 1 records I0428 11:10:10.709896 1 tasks_processing.go:74] worker 32 stopped. I0428 11:10:10.709948 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0428 11:10:10.709960 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 58.926278ms to process 1 records I0428 11:10:10.715830 1 tasks_processing.go:74] worker 53 stopped. I0428 11:10:10.716035 1 controller.go:254] Source scaController *sca.Controller is not ready I0428 11:10:10.716056 1 controller.go:254] Source clusterTransferController *clustertransfer.Controller is not ready I0428 11:10:10.716061 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0428 11:10:10.716065 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0428 11:10:10.716069 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0428 11:10:10.716111 1 controller.go:531] The operator is still being initialized I0428 11:10:10.716123 1 controller.go:554] The operator is healthy I0428 11:10:10.716396 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=bf40699e6737125ed9d3eab699f9aa50f9c3ae393c5968aaaf45e6cbed6f0a7e I0428 11:10:10.716618 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=8285f7fbf558a56f8008730668dfe94e6bcf02ea2c48e10f7ef7ca372be42c55 I0428 11:10:10.716629 1 gather.go:177] gatherer "clusterconfig" function "crds" took 65.717002ms to process 2 records I0428 11:10:10.720456 1 configmapobserver.go:84] configmaps "insights-config" not found I0428 11:10:10.720603 1 tasks_processing.go:74] worker 51 stopped. I0428 11:10:10.720821 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=c4ca8ce43c437d0d0aadf21770200c680a56f0e88dce1e8929cbb8d13e9cad60 I0428 11:10:10.720968 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=cffe5e0fde2c84344bf976fd17625dde9c28a1290f0ed8a1ee977ab3ae4b453d I0428 11:10:10.720981 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 67.587911ms to process 2 records E0428 11:10:10.721001 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0428 11:10:10.721024 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0428 11:10:10.721034 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 70.08072ms to process 1 records I0428 11:10:10.721044 1 tasks_processing.go:74] worker 63 stopped. I0428 11:10:10.721206 1 tasks_processing.go:74] worker 22 stopped. I0428 11:10:10.721261 1 recorder.go:75] Recording config/olm_operators with fingerprint=dd1fb42f46e93401b1511dfaa83708712170f428912c36a3ff289941db85ea59 I0428 11:10:10.721274 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 72.454602ms to process 1 records I0428 11:10:10.731473 1 tasks_processing.go:74] worker 16 stopped. I0428 11:10:10.731498 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 78.451716ms to process 0 records I0428 11:10:10.740161 1 tasks_processing.go:74] worker 14 stopped. I0428 11:10:10.740179 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 88.415142ms to process 0 records I0428 11:10:10.741204 1 tasks_processing.go:74] worker 35 stopped. I0428 11:10:10.743342 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-66dbbb6468-zh5kn with fingerprint=daaf2ffb6870571511c7484a6a1e12dcb457f0e4b37117ebb4d46cbad5009ceb I0428 11:10:10.743465 1 recorder.go:75] Recording config/running_containers with fingerprint=7fbef82e01d5686a3fe6b97afa57e26cbbf579fcc5a95515469fcc2a98f6bc7f I0428 11:10:10.743480 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 88.300426ms to process 2 records I0428 11:10:10.746905 1 tasks_processing.go:74] worker 43 stopped. E0428 11:10:10.746918 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0428 11:10:10.746927 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2pv477t4c9srti67smm59tsf841pmq8b-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2pv477t4c9srti67smm59tsf841pmq8b-primary-cert-bundle-secret" not found I0428 11:10:10.747006 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=a00c5e9be15f2a49d94bcb45785cb65e2cdce383ea271a95386aa3c754cf7110 I0428 11:10:10.747025 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 98.434197ms to process 1 records I0428 11:10:10.753469 1 tasks_processing.go:74] worker 62 stopped. E0428 11:10:10.753485 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0428 11:10:10.753492 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0428 11:10:10.753495 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0428 11:10:10.753505 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=ad286723f58bdcfc37aeba3ec5b4110c08e1af59cd34d14b4bfaab02d18e4856 I0428 11:10:10.753542 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0428 11:10:10.753550 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0428 11:10:10.753570 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=2ab5ab7b1b10d7fcf1197bb24dea7c90f400e4effc18c7356873209d54fdf84b I0428 11:10:10.753583 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0428 11:10:10.753620 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0428 11:10:10.753628 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0428 11:10:10.753634 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 101.291638ms to process 7 records I0428 11:10:10.766233 1 cluster_transfer.go:102] no available accepted cluster transfer I0428 11:10:10.766255 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0428 11:10:10.771587 1 tasks_processing.go:74] worker 50 stopped. I0428 11:10:10.771880 1 recorder.go:75] Recording config/version with fingerprint=eb711a3f8916d2ded469c47afc76e477c68408251cb46f1518cb08b05041674a I0428 11:10:10.771895 1 recorder.go:75] Recording config/id with fingerprint=f13f72656252226c6f77244849093c9f71c5787f6a79763b3da169f0557cbca1 I0428 11:10:10.771903 1 gather.go:177] gatherer "clusterconfig" function "version" took 122.606065ms to process 2 records I0428 11:10:10.772665 1 base_controller.go:82] Caches are synced for LoggingSyncer I0428 11:10:10.772681 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0428 11:10:10.785160 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0428 11:10:10.785265 1 gather_cluster_operator_pods_and_events.go:121] Found 0 pods with 0 containers I0428 11:10:10.785288 1 tasks_processing.go:74] worker 17 stopped. I0428 11:10:10.785301 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 137.224783ms to process 0 records I0428 11:10:10.814161 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0428 11:10:10.818269 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0428 11:10:11.007028 1 sca.go:266] etc-pki-entitlement secret successfully created I0428 11:10:11.007054 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated W0428 11:10:11.697395 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0428 11:10:12.697441 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0428 11:10:13.156309 1 tasks_processing.go:74] worker 10 stopped. I0428 11:10:13.156433 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=148a70a5d184d8d18258674932e902ed09398f286b911056298f9f1e224c1587 I0428 11:10:13.156482 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0428 11:10:13.156517 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=e468f2beb1cbeb423c740c06940ba509d27618c98a6401c85df37740ea57f247 I0428 11:10:13.156527 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0428 11:10:13.156548 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=4f5e503c524fb6931ebb6908dcffcb0e83ff89c453961d45bf6863c07ffee7f2 I0428 11:10:13.156563 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0428 11:10:13.156584 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=1b1b280a002ed36099fbeb3d2258183dbe9c1b61049be09a957a08c01334a1c6 I0428 11:10:13.156612 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=7edef32429f9486ba8f7876cda4b8c66318daeb60199262409e059e273710a39 I0428 11:10:13.156635 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=49aee72e13ebce052b5e6085225c344624afc221f7a914c5cc52cea7623a0289 I0428 11:10:13.156665 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=6033e42c7a27548c8c74a25a2e79daf37482db3775b89a1ee78f13015547f86a I0428 11:10:13.156949 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=7817c0e0a92c2bb63f6e06c572d0f5e7ccf8f3a9eaade16163a32ee69915e9fc I0428 11:10:13.156969 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=1b591adacba791f96b9e2cf5659f3233543c4558f91ab069de7f264b56504224 I0428 11:10:13.156977 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0428 11:10:13.156991 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=f364652aa78fe5149b1f7f252da46696e30dd3428687683de2b7eca8700e308a I0428 11:10:13.157002 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0428 11:10:13.157017 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=5d46e7105c0521903b03aed034f13e46837bf3c9bd4a1fb8564692b3fcb53e78 I0428 11:10:13.157026 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0428 11:10:13.157046 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=f01c61e3236e056174adbe742edb8898017151580cf906648e7b03548fc64c44 I0428 11:10:13.157055 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0428 11:10:13.157075 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=7aa7f180fcf64209111bb9fee995da7c190ed5e2489624b7e119eb134f8ad76d I0428 11:10:13.157180 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=86314b4a4bdd661d5457f6876c38e993e7d31baef827899d9ef1eb17dff0c680 I0428 11:10:13.157190 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0428 11:10:13.157197 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0428 11:10:13.157219 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0428 11:10:13.157243 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=609d75c56259d9a1efe68df2f84e9ac65a0797266ae52da3f63db20ce2c742ea I0428 11:10:13.157266 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=91bcb18769367d487db3df1a05910cbc754890eec94ee4eb4a965353aa7eaef6 I0428 11:10:13.157277 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0428 11:10:13.157293 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=e5b92d0cc896edbe09695ad307d88053d3401932830599536fd68377b87261d0 I0428 11:10:13.157304 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0428 11:10:13.157321 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=b957cada89dcdef4a319eba4174014b974f6e0cb5244bdf756880fcfae764cf2 I0428 11:10:13.157330 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0428 11:10:13.157342 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=39f3a73926b3cd42c66b391e66ecd469a87dbfc2d44ec6c59c4965e4ebf7156b I0428 11:10:13.157360 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=52549d9fdc6e27de4c8fbbeb3bbd400c0ee76c2b22b50d218dfe66ec68e7b8ef I0428 11:10:13.157374 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=3f6b9dedb9f48693d321c83ee9c67eaad1964f8899e2686f68396ba091c829e5 I0428 11:10:13.157394 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=86f6c268054d6549023d643711991f7df1f6d6ad3de81513c3241028f79149f8 I0428 11:10:13.157403 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0428 11:10:13.157425 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=b8295d8d011b3fc00ea903e89081dc2325b5e11f6f1c1e57174b4d9c962a3fb0 I0428 11:10:13.157442 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0428 11:10:13.157448 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0428 11:10:13.157455 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.503316741s to process 39 records W0428 11:10:13.697143 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0428 11:10:14.697434 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0428 11:10:15.697188 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0428 11:10:15.697236 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0428 11:10:15.697257 1 tasks_processing.go:74] worker 9 stopped. E0428 11:10:15.697265 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0428 11:10:15.697278 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0428 11:10:15.697289 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.046080293s to process 1 records I0428 11:10:23.508853 1 tasks_processing.go:74] worker 57 stopped. I0428 11:10:23.508901 1 recorder.go:75] Recording config/installplans with fingerprint=95dfd5f33a9a46199239158a0ec0183b629818eca24ae41f4d706d7a3d604aec I0428 11:10:23.508914 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.858894196s to process 1 records I0428 11:10:24.257444 1 tasks_processing.go:74] worker 47 stopped. I0428 11:10:24.257740 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=796bd9f20e825fd32af435d21e60c10c3ec4910fa1211c5a6338f5b81c3f8071 I0428 11:10:24.257759 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.608703604s to process 1 records E0428 11:10:24.257825 1 periodic.go:254] "Unhandled Error" err="clusterconfig failed after 13.61s with: function \"tsdb_status\" failed with an error, function \"metrics\" failed with an error, function \"active_alerts\" failed with an error, function \"silenced_alerts\" failed with an error, function \"support_secret\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"machines\" failed with an error, function \"machine_configs\" failed with an error, function \"ingress_certificates\" failed with an error, function \"config_maps\" failed with an error, function \"dvo_metrics\" failed with an error" I0428 11:10:24.258940 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "tsdb_status" failed with an error, function "metrics" failed with an error, function "active_alerts" failed with an error, function "silenced_alerts" failed with an error, function "support_secret" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machine_healthchecks" failed with an error, function "machines" failed with an error, function "machine_configs" failed with an error, function "ingress_certificates" failed with an error, function "config_maps" failed with an error, function "dvo_metrics" failed with an error I0428 11:10:24.258957 1 periodic.go:216] Running workloads gatherer I0428 11:10:24.258977 1 tasks_processing.go:45] number of workers: 2 I0428 11:10:24.258992 1 tasks_processing.go:69] worker 1 listening for tasks. I0428 11:10:24.258999 1 tasks_processing.go:71] worker 1 working on workload_info task. I0428 11:10:24.259004 1 tasks_processing.go:69] worker 0 listening for tasks. I0428 11:10:24.259083 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0428 11:10:24.265440 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-135.ec2.internal... I0428 11:10:24.265461 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-140.ec2.internal... I0428 11:10:24.265442 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-109.ec2.internal... I0428 11:10:24.286192 1 tasks_processing.go:74] worker 0 stopped. I0428 11:10:24.286212 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 27.08515ms to process 0 records I0428 11:10:28.910699 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 5s I0428 11:10:28.934275 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0428 11:10:28.946511 1 gather_workloads_info.go:387] No image sha256:a043239802b3eb8b323d285193d2527fad0ecec98ca91d188a3472a2fac8ae04 (14ms) I0428 11:10:28.955682 1 gather_workloads_info.go:387] No image sha256:730d1b6988025bef0daa3a9a5d8467ec4a26b0382cc52f91c3375b4590d3518a (9ms) I0428 11:10:28.964574 1 gather_workloads_info.go:387] No image sha256:52db05084f2a2e7d7bf4040e4865df88ec4074efbbe59a78d861be2c5ab2aa40 (9ms) I0428 11:10:28.973552 1 gather_workloads_info.go:387] No image sha256:53ff8efdd8f563421ebc1859ab021a3bbc1306f82d4ff5d9c21bc60ec5f57898 (9ms) I0428 11:10:28.982258 1 gather_workloads_info.go:387] No image sha256:f68c0da8268f23078f5f760a91a3c8212f2195acf18c231f6e034207a67adeec (9ms) I0428 11:10:28.991157 1 gather_workloads_info.go:387] No image sha256:ab60623bb32f7e75fca71ef65137731cae347a21c7a4091dfd583fa00732721c (9ms) I0428 11:10:28.999899 1 gather_workloads_info.go:387] No image sha256:572ebf7c7fab1a518582f3ca46c866730423cce6e8a19f614dc15fd61bf0c65a (9ms) I0428 11:10:29.008024 1 gather_workloads_info.go:387] No image sha256:5940cc34b322325618929b1e0eb3864dae73d54c8f5b474e218dfb28123a61c7 (8ms) I0428 11:10:29.019488 1 gather_workloads_info.go:387] No image sha256:30e597ec5d6bb96ff70a4f8688c748b659cd4fd5d73d222e8701821d236795c5 (11ms) I0428 11:10:29.028172 1 gather_workloads_info.go:387] No image sha256:48e883a3932aea9457f55cd4628d21397d429d4828f42f95d1c903d9d5395bde (9ms) I0428 11:10:29.042064 1 gather_workloads_info.go:387] No image sha256:48f5559439bea030a6423652da448c25fff0c62ba293eb521843c1a8eb89ddb0 (14ms) I0428 11:10:29.143510 1 gather_workloads_info.go:387] No image sha256:5d6a2b8d2508c7da71192661e7c0dcf1335f48b6336a20189abffbe79eeff962 (101ms) I0428 11:10:29.243931 1 gather_workloads_info.go:387] No image sha256:5c6d21c3f97366bc7ab57031cc027b67405a684bd804ce364ed5998b0685eaca (100ms) I0428 11:10:29.343281 1 gather_workloads_info.go:387] No image sha256:9d6d63dce784df5863e6a0ba4c11317a26c9d756d4e62d47759c97190a71cce5 (99ms) I0428 11:10:29.443344 1 gather_workloads_info.go:387] No image sha256:1eb6a6ee3c66ac15be6bc21bb6015c595c11c15b9dfd21bd9f5ce7f7b990ebaa (100ms) I0428 11:10:29.543387 1 gather_workloads_info.go:387] No image sha256:ae7d3453fd734ecc865e5f9bb16f29244ebffe6291b77e1d4e496f71eb053174 (100ms) I0428 11:10:29.644029 1 gather_workloads_info.go:387] No image sha256:0b2e6536a0ca3afc5e64a218087a6287c74b77a359075a385f1949ca5bd51b22 (101ms) I0428 11:10:29.743745 1 gather_workloads_info.go:387] No image sha256:23204877f166455ce8f9de9b91a467eeda576f172d5f96124a68aa986c4775d6 (100ms) I0428 11:10:29.844160 1 gather_workloads_info.go:387] No image sha256:5b6212b8f539f08e78417d8a4b7485ca0b4e7927cacd7b752742a28841bc8ccd (100ms) I0428 11:10:29.944060 1 gather_workloads_info.go:387] No image sha256:5808401268394502d335281ea1a294b07210461b986b58f91d7d1f29c0029c6d (100ms) I0428 11:10:30.043621 1 gather_workloads_info.go:387] No image sha256:8f881f6f0498eaba860d711dc8e9dc066db8728f22abedb82e1dc9a8e1c30046 (100ms) I0428 11:10:30.143646 1 gather_workloads_info.go:387] No image sha256:5aaea0419169e55832cc27acfe0fe3b9513a343d6bdf71d3da1575ed322245d0 (100ms) I0428 11:10:30.244973 1 gather_workloads_info.go:387] No image sha256:25c148fd380b1a9db3f6039d2e0eabc489a954921452391390ba9192b2325678 (101ms) I0428 11:10:30.344197 1 gather_workloads_info.go:387] No image sha256:000105ef5150e7079b90a613fb9e6193e2a6ef9b1908d2dce44f2395d4fd070f (99ms) I0428 11:10:30.443061 1 gather_workloads_info.go:387] No image sha256:0a4dfb8d4c1b3849319d45b4c54dff26a7238a2c08fcaa121f93073e95ab12e8 (99ms) I0428 11:10:30.543026 1 gather_workloads_info.go:387] No image sha256:637d41f067a5239096fc22b135181cda5113da833f1370e7a73965e83792e93a (100ms) I0428 11:10:30.643885 1 gather_workloads_info.go:387] No image sha256:3bba1358d4a0ae878ff491c0c2cbfffe60649e110b40342b878fe8fa332f8858 (101ms) I0428 11:10:30.743582 1 gather_workloads_info.go:387] No image sha256:084aa9b0f8a6d478549dc384d4e66da13ee9b25cc98531da861cc19dee2a9e8f (100ms) I0428 11:10:30.843764 1 gather_workloads_info.go:387] No image sha256:08c5a78c8a5af04c549e2273aaf4bb452a75bf038d68aa9d01bb2aff66c30e90 (100ms) I0428 11:10:30.949835 1 gather_workloads_info.go:387] No image sha256:6814f5fced0ef219d06374011c68a11a3da788a764a00a69ade435466d9ee240 (106ms) I0428 11:10:31.044202 1 gather_workloads_info.go:387] No image sha256:bdeb1c455cffccdb6c90db238e9d371de06340b1d6105d47f66d87293f878ac1 (94ms) I0428 11:10:31.144142 1 gather_workloads_info.go:387] No image sha256:15928a5b803fa00781b408dbc4c3bfa7c5287b18584ba31f817def8a4430c725 (100ms) I0428 11:10:31.243494 1 gather_workloads_info.go:387] No image sha256:5ac9c549d65fc1d8bc900773bebee43e9192bcec1bb5fa46afb4597230c16ac7 (99ms) I0428 11:10:31.347313 1 gather_workloads_info.go:387] No image sha256:0acfe677ed58031f8381cb8a4f2d8917ec4cfa95485e2a46529dffae54c75ace (104ms) I0428 11:10:31.444007 1 gather_workloads_info.go:387] No image sha256:673ebc8cc22c56c8f410e011b2fa950d28cf7b6420e17fdb6580d6cb10523384 (97ms) I0428 11:10:31.543368 1 gather_workloads_info.go:387] No image sha256:15e1b5f763553e86d0230a751b0783e3b990438fd07cf96eed164bccb72ed564 (99ms) I0428 11:10:31.643334 1 gather_workloads_info.go:387] No image sha256:d64bea34bf3e1bb0b3a701c3ff14e66665afc1b050f28124ad7e6888eaec3a81 (100ms) I0428 11:10:31.744189 1 gather_workloads_info.go:387] No image sha256:3380fecebaeee139c1607fbcd9346cebc92806679568e2244a8905ef0f52aad2 (101ms) I0428 11:10:31.843363 1 gather_workloads_info.go:387] No image sha256:55b1db6038c5beaed54c626e3343b7a8589cc0be8dc41d1a66b4deab766ff520 (99ms) I0428 11:10:31.944058 1 gather_workloads_info.go:387] No image sha256:9ef6dc311b880a6442b7a3a19006ddd955c7da36850bf000f5c9b68fd0862dd4 (101ms) I0428 11:10:32.043519 1 gather_workloads_info.go:387] No image sha256:89277d8d4560d71db88c2dcc67c992a24544ca21810920b609c1d49d53b4a287 (99ms) I0428 11:10:32.144393 1 gather_workloads_info.go:387] No image sha256:15677f0b70e6aa2dfaf088e45fc1a425c22bf6fda326b8116f87e88b6694dfab (101ms) I0428 11:10:32.243884 1 gather_workloads_info.go:387] No image sha256:934d8e8c50f3c609b8eea80d1051111fe3d066fe8c65c79572072ae55fcb0a86 (99ms) I0428 11:10:32.342824 1 gather_workloads_info.go:387] No image sha256:6c6d412ccf0c3d6a14645a79df6c4658c0f8725d7954cb83c3d2f0116499345a (99ms) I0428 11:10:32.444066 1 gather_workloads_info.go:387] No image sha256:17aefdc2f4752c23dc8f0947bb01fc34635822eeb18f2d8a5eddf8211b84d450 (101ms) I0428 11:10:32.543067 1 gather_workloads_info.go:387] No image sha256:5ff204630794311b0b37fa7b197b933ad85d76a481bf7cdb3bcbada08f0cdcbf (99ms) I0428 11:10:32.643831 1 gather_workloads_info.go:387] No image sha256:875c77e5d144f03fb91d8cee0259f6966683ca88d1bf818dbf4652c16b70312c (101ms) I0428 11:10:32.643867 1 tasks_processing.go:74] worker 1 stopped. I0428 11:10:32.644151 1 recorder.go:75] Recording config/workload_info with fingerprint=75a5eed3230993927dbd3683949b580040a0f384e4e686433f9008a6d3285b91 I0428 11:10:32.644167 1 gather.go:177] gatherer "workloads" function "workload_info" took 8.384860437s to process 1 records I0428 11:10:32.644183 1 periodic.go:263] Periodic gather workloads completed in 8.385s I0428 11:10:32.644190 1 controllerstatus.go:80] name=periodic-workloads healthy=true reason= message= I0428 11:10:32.644194 1 periodic.go:216] Running conditional gatherer I0428 11:10:32.649819 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.21.9/gathering_rules I0428 11:10:32.845192 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer E0428 11:10:32.847746 1 conditional_gatherer.go:324] unable to update alerts cache: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/query?match%5B%5D=ALERTS%7Balertstate%3D%22firing%22%7D&query=ALERTS": dial tcp 172.30.249.241:9091: connect: connection refused I0428 11:10:32.847824 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0428 11:10:32.854028 1 conditional_gatherer.go:392] cluster version is '4.21.9' E0428 11:10:32.854046 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0428 11:10:32.854052 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0428 11:10:32.854056 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0428 11:10:32.854059 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0428 11:10:32.854062 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0428 11:10:32.854065 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0428 11:10:32.854068 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0428 11:10:32.854071 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0428 11:10:32.854074 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0428 11:10:32.854078 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0428 11:10:32.854092 1 tasks_processing.go:45] number of workers: 3 I0428 11:10:32.854111 1 tasks_processing.go:69] worker 2 listening for tasks. I0428 11:10:32.854115 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0428 11:10:32.854121 1 tasks_processing.go:69] worker 0 listening for tasks. I0428 11:10:32.854135 1 tasks_processing.go:69] worker 1 listening for tasks. I0428 11:10:32.854150 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0428 11:10:32.854153 1 tasks_processing.go:74] worker 1 stopped. I0428 11:10:32.854161 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0428 11:10:32.854198 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=9fafdcbf373f1f367f1d69551ea59114592cce1681189be0f3dde774e0530661 I0428 11:10:32.854208 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 847ns to process 1 records I0428 11:10:32.854241 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0428 11:10:32.854248 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.448µs to process 1 records I0428 11:10:32.854271 1 tasks_processing.go:74] worker 0 stopped. I0428 11:10:32.854465 1 tasks_processing.go:74] worker 2 stopped. I0428 11:10:32.854483 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 285.008µs to process 0 records I0428 11:10:32.854507 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0428 11:10:32.854533 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0428 11:10:32.880930 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=ed58024b9d2f86dc3b77bd9aeab5ba2a03e20aad9f17eb107721e54babda9692 I0428 11:10:32.881073 1 diskrecorder.go:70] Writing 109 records to /var/lib/insights-operator/insights-2026-04-28-111032.tar.gz I0428 11:10:32.887626 1 diskrecorder.go:51] Wrote 109 records to disk in 6ms I0428 11:10:32.887662 1 periodic.go:285] Gathering cluster info every 2h0m0s I0428 11:10:32.887677 1 periodic.go:286] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0428 11:11:40.648290 1 diskrecorder.go:223] Found files to send: insights-2026-04-28-111032.tar.gz I0428 11:11:40.648323 1 insightsuploader.go:150] Checking archives to upload periodically every 16m19.635013705s I0428 11:11:40.648332 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0428 11:11:40.659005 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0428 11:11:40.904579 1 requests.go:87] Successfully reported id=2026-04-28T11:11:40Z x-rh-insights-request-id=90734c276dc24d9195ec1b03337fe9e0, wrote=48462 I0428 11:11:40.904646 1 insightsuploader.go:187] Uploaded report successfully in 256.30195ms I0428 11:11:40.904679 1 controller.go:129] Initializing last reported time to 2026-04-28T11:11:40Z I0428 11:11:40.904742 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0428 11:11:40.904752 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0428 11:11:40.904759 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0428 11:11:40.911033 1 controller.go:554] The operator is healthy I0428 11:12:10.679682 1 controller.go:554] The operator is healthy I0428 11:12:42.076493 1 insightsreport.go:137] Pulling report from smart-proxy I0428 11:12:42.076521 1 insightsreport.go:149] Retrieving report I0428 11:12:42.083738 1 requests.go:110] Retrieving report for cluster: 5f344aa7-fa06-426e-b3ff-7265a8470db8 I0428 11:12:42.083748 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/5f344aa7-fa06-426e-b3ff-7265a8470db8/reports I0428 11:12:42.087011 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/5f344aa7-fa06-426e-b3ff-7265a8470db8/reports I0428 11:12:42.434698 1 insightsreport.go:184] Report retrieved I0428 11:12:42.460825 1 insightsreport.go:239] Report retrieved correctly I0428 11:14:10.681887 1 controller.go:554] The operator is healthy I0428 11:15:10.636923 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0428 11:15:10.645591 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0428 11:15:10.645611 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0428 11:15:10.649933 1 secretconfigobserver.go:119] support secret does not exist I0428 11:16:10.681496 1 controller.go:554] The operator is healthy I0428 11:18:10.678744 1 controller.go:554] The operator is healthy I0428 11:20:10.650989 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0428 11:20:10.829037 1 configmapobserver.go:84] configmaps "insights-config" not found I0428 11:20:10.829189 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0428 11:20:10.829211 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0428 11:20:10.847943 1 controller.go:554] The operator is healthy I0428 11:20:11.236344 1 secretconfigobserver.go:119] support secret does not exist I0428 11:22:10.678959 1 controller.go:554] The operator is healthy I0428 11:24:10.679149 1 controller.go:554] The operator is healthy