I0424 19:03:05.806686 1 cmd.go:253] Using service-serving-cert provided certificates I0424 19:03:05.807153 1 observer_polling.go:159] Starting file observer I0424 19:03:06.529188 1 operator.go:76] Starting insights-operator v0.0.0-master+$Format:%H$ I0424 19:03:06.529423 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0424 19:03:06.529961 1 secure_serving.go:57] Forcing use of http/1.1 only W0424 19:03:06.529980 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0424 19:03:06.529984 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0424 19:03:06.529988 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0424 19:03:06.529990 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0424 19:03:06.529995 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0424 19:03:06.529997 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0424 19:03:06.530076 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0424 19:03:06.536066 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0424 19:03:06.536076 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0424 19:03:06.536081 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0424 19:03:06.536101 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0424 19:03:06.536114 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0424 19:03:06.536101 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0424 19:03:06.536367 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0424 19:03:06.536421 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"a2ed3190-084a-457b-85ac-ca0dfdac309f", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "HyperShiftOnlyDynamicResourceAllocation", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryption", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLM", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "OpenShiftPodSecurityAdmission", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}} I0424 19:03:06.536402 1 operator.go:141] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNSInstall GatewayAPI GatewayAPIController HighlyAvailableArbiter HyperShiftOnlyDynamicResourceAllocation ImageStreamImportMode ImageVolume KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS ManagedBootImagesAzure ManagedBootImagesvSphere MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages PreconfiguredUDNAddresses ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SigstoreImageVerification SigstoreImageVerificationPKI StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSDualStackInstall AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureDualStackInstall AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement CBORServingAndStorage CRDCompatibilityRequirementOperator ClientsAllowCBOR ClientsPreferCBOR ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterAPIMachineManagement ClusterAPIMachineManagementVSphere ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud EtcdBackendQuota EventTTL EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall GCPDualStackInstall ImageModeStatusReporting IngressControllerDynamicConfigurationManager InsightsConfig InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryption KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesCPMS MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutableCSINodeAllocatableCount MutatingAdmissionPolicy NewOLM NewOLMBoxCutterRuntime NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterInstall NutanixMultiSubnets OSStreams OVNObservability OnPremDNSRecords OpenShiftPodSecurityAdmission ProvisioningRequestAvailable SELinuxMount ShortCertRotation SignatureStores TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeGroupSnapshot] I0424 19:03:06.536672 1 secure_serving.go:213] Serving securely on [::]:8443 I0424 19:03:06.536703 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0424 19:03:06.542596 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0424 19:03:06.542639 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0424 19:03:06.542681 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0424 19:03:06.547992 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0424 19:03:06.548014 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0424 19:03:06.553128 1 secretconfigobserver.go:119] support secret does not exist I0424 19:03:06.558349 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0424 19:03:06.562781 1 secretconfigobserver.go:119] support secret does not exist I0424 19:03:06.565802 1 recorder.go:176] Pruning old reports every 7h40m17s, max age is 288h0m0s I0424 19:03:06.571438 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0424 19:03:06.571463 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0424 19:03:06.571442 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0424 19:03:06.571488 1 insightsreport.go:296] Starting report retriever I0424 19:03:06.571492 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0424 19:03:06.571490 1 periodic.go:216] Running clusterconfig gatherer I0424 19:03:06.571542 1 tasks_processing.go:45] number of workers: 64 I0424 19:03:06.571573 1 tasks_processing.go:69] worker 0 listening for tasks. I0424 19:03:06.571588 1 tasks_processing.go:69] worker 4 listening for tasks. I0424 19:03:06.571588 1 tasks_processing.go:69] worker 1 listening for tasks. I0424 19:03:06.571597 1 tasks_processing.go:69] worker 3 listening for tasks. I0424 19:03:06.571595 1 tasks_processing.go:69] worker 2 listening for tasks. I0424 19:03:06.571604 1 tasks_processing.go:69] worker 16 listening for tasks. I0424 19:03:06.571605 1 tasks_processing.go:69] worker 27 listening for tasks. I0424 19:03:06.571613 1 tasks_processing.go:69] worker 30 listening for tasks. I0424 19:03:06.571614 1 tasks_processing.go:69] worker 31 listening for tasks. I0424 19:03:06.571619 1 tasks_processing.go:69] worker 5 listening for tasks. I0424 19:03:06.571614 1 tasks_processing.go:69] worker 17 listening for tasks. I0424 19:03:06.571621 1 tasks_processing.go:69] worker 18 listening for tasks. I0424 19:03:06.571625 1 tasks_processing.go:69] worker 32 listening for tasks. I0424 19:03:06.571626 1 tasks_processing.go:69] worker 42 listening for tasks. I0424 19:03:06.571633 1 tasks_processing.go:71] worker 31 working on node_logs task. I0424 19:03:06.571635 1 tasks_processing.go:71] worker 32 working on feature_gates task. I0424 19:03:06.571639 1 tasks_processing.go:69] worker 8 listening for tasks. I0424 19:03:06.571639 1 tasks_processing.go:69] worker 29 listening for tasks. I0424 19:03:06.571647 1 tasks_processing.go:71] worker 1 working on sap_config task. I0424 19:03:06.571652 1 tasks_processing.go:69] worker 9 listening for tasks. I0424 19:03:06.571652 1 tasks_processing.go:71] worker 29 working on clusterroles task. I0424 19:03:06.571654 1 tasks_processing.go:69] worker 63 listening for tasks. I0424 19:03:06.571662 1 tasks_processing.go:69] worker 37 listening for tasks. I0424 19:03:06.571666 1 tasks_processing.go:69] worker 46 listening for tasks. I0424 19:03:06.571671 1 tasks_processing.go:69] worker 57 listening for tasks. I0424 19:03:06.571667 1 tasks_processing.go:69] worker 22 listening for tasks. I0424 19:03:06.571678 1 tasks_processing.go:69] worker 58 listening for tasks. I0424 19:03:06.571659 1 tasks_processing.go:69] worker 10 listening for tasks. I0424 19:03:06.571667 1 tasks_processing.go:69] worker 11 listening for tasks. I0424 19:03:06.571685 1 tasks_processing.go:69] worker 40 listening for tasks. I0424 19:03:06.571619 1 tasks_processing.go:71] worker 27 working on openstack_dataplanedeployments task. I0424 19:03:06.571689 1 tasks_processing.go:69] worker 20 listening for tasks. I0424 19:03:06.571691 1 tasks_processing.go:69] worker 49 listening for tasks. I0424 19:03:06.571629 1 tasks_processing.go:69] worker 6 listening for tasks. I0424 19:03:06.571628 1 tasks_processing.go:71] worker 17 working on node_features task. I0424 19:03:06.571640 1 tasks_processing.go:71] worker 0 working on infrastructures task. I0424 19:03:06.571702 1 tasks_processing.go:71] worker 5 working on install_plans task. I0424 19:03:06.571630 1 tasks_processing.go:71] worker 18 working on openstack_dataplanenodesets task. I0424 19:03:06.571845 1 tasks_processing.go:69] worker 15 listening for tasks. I0424 19:03:06.571860 1 tasks_processing.go:69] worker 13 listening for tasks. I0424 19:03:06.571860 1 tasks_processing.go:69] worker 25 listening for tasks. I0424 19:03:06.571637 1 tasks_processing.go:69] worker 56 listening for tasks. I0424 19:03:06.571643 1 tasks_processing.go:71] worker 4 working on operators_pods_and_events task. I0424 19:03:06.571866 1 tasks_processing.go:69] worker 52 listening for tasks. I0424 19:03:06.571873 1 tasks_processing.go:69] worker 28 listening for tasks. I0424 19:03:06.571635 1 tasks_processing.go:71] worker 42 working on cost_management_metrics_configs task. I0424 19:03:06.571879 1 tasks_processing.go:69] worker 51 listening for tasks. I0424 19:03:06.571887 1 tasks_processing.go:71] worker 3 working on overlapping_namespace_uids task. I0424 19:03:06.571882 1 tasks_processing.go:71] worker 2 working on storage_cluster task. I0424 19:03:06.571902 1 tasks_processing.go:69] worker 53 listening for tasks. I0424 19:03:06.571920 1 tasks_processing.go:69] worker 23 listening for tasks. I0424 19:03:06.571922 1 tasks_processing.go:69] worker 21 listening for tasks. I0424 19:03:06.571608 1 tasks_processing.go:69] worker 33 listening for tasks. I0424 19:03:06.571645 1 tasks_processing.go:71] worker 8 working on machine_healthchecks task. I0424 19:03:06.571935 1 tasks_processing.go:71] worker 16 working on nodenetworkconfigurationpolicies task. I0424 19:03:06.571939 1 tasks_processing.go:69] worker 14 listening for tasks. I0424 19:03:06.571622 1 tasks_processing.go:71] worker 30 working on mutating_webhook_configurations task. I0424 19:03:06.571958 1 tasks_processing.go:69] worker 24 listening for tasks. I0424 19:03:06.571635 1 tasks_processing.go:69] worker 34 listening for tasks. I0424 19:03:06.571674 1 tasks_processing.go:69] worker 38 listening for tasks. I0424 19:03:06.571988 1 tasks_processing.go:69] worker 54 listening for tasks. I0424 19:03:06.571676 1 tasks_processing.go:69] worker 47 listening for tasks. I0424 19:03:06.571679 1 tasks_processing.go:69] worker 39 listening for tasks. I0424 19:03:06.571684 1 tasks_processing.go:69] worker 19 listening for tasks. I0424 19:03:06.571999 1 tasks_processing.go:69] worker 61 listening for tasks. I0424 19:03:06.571866 1 tasks_processing.go:69] worker 26 listening for tasks. I0424 19:03:06.571684 1 tasks_processing.go:69] worker 48 listening for tasks. I0424 19:03:06.572011 1 tasks_processing.go:69] worker 55 listening for tasks. I0424 19:03:06.572021 1 tasks_processing.go:69] worker 59 listening for tasks. I0424 19:03:06.572021 1 tasks_processing.go:69] worker 62 listening for tasks. I0424 19:03:06.571624 1 tasks_processing.go:69] worker 41 listening for tasks. I0424 19:03:06.572030 1 tasks_processing.go:69] worker 60 listening for tasks. I0424 19:03:06.572038 1 tasks_processing.go:71] worker 22 working on config_maps task. I0424 19:03:06.572043 1 tasks_processing.go:71] worker 60 working on metrics task. I0424 19:03:06.572070 1 tasks_processing.go:69] worker 50 listening for tasks. I0424 19:03:06.572082 1 tasks_processing.go:71] worker 50 working on machine_sets task. I0424 19:03:06.572093 1 tasks_processing.go:71] worker 37 working on sap_pods task. I0424 19:03:06.572109 1 tasks_processing.go:71] worker 46 working on qemu_kubevirt_launcher_logs task. I0424 19:03:06.571644 1 tasks_processing.go:69] worker 43 listening for tasks. I0424 19:03:06.572124 1 tasks_processing.go:71] worker 21 working on oauths task. I0424 19:03:06.572131 1 tasks_processing.go:71] worker 39 working on monitoring_persistent_volumes task. I0424 19:03:06.572144 1 tasks_processing.go:71] worker 9 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0424 19:03:06.572156 1 tasks_processing.go:71] worker 15 working on storage_classes task. I0424 19:03:06.572173 1 tasks_processing.go:71] worker 57 working on openstack_controlplanes task. I0424 19:03:06.572176 1 tasks_processing.go:71] worker 43 working on dvo_metrics task. I0424 19:03:06.571652 1 tasks_processing.go:69] worker 36 listening for tasks. I0424 19:03:06.571651 1 tasks_processing.go:69] worker 44 listening for tasks. I0424 19:03:06.572271 1 tasks_processing.go:71] worker 63 working on ingress task. I0424 19:03:06.572277 1 tasks_processing.go:71] worker 44 working on machine_configs task. I0424 19:03:06.572311 1 tasks_processing.go:71] worker 34 working on container_runtime_configs task. I0424 19:03:06.572404 1 tasks_processing.go:71] worker 55 working on authentication task. I0424 19:03:06.572451 1 tasks_processing.go:71] worker 40 working on service_accounts task. I0424 19:03:06.572504 1 tasks_processing.go:71] worker 33 working on machines task. I0424 19:03:06.571659 1 tasks_processing.go:69] worker 45 listening for tasks. I0424 19:03:06.572556 1 tasks_processing.go:71] worker 14 working on tsdb_status task. I0424 19:03:06.572572 1 tasks_processing.go:71] worker 19 working on version task. I0424 19:03:06.572602 1 tasks_processing.go:71] worker 38 working on openshift_machine_api_events task. I0424 19:03:06.572681 1 tasks_processing.go:71] worker 49 working on pdbs task. I0424 19:03:06.572698 1 tasks_processing.go:71] worker 11 working on openstack_version task. I0424 19:03:06.572699 1 tasks_processing.go:71] worker 58 working on certificate_signing_requests task. I0424 19:03:06.571633 1 tasks_processing.go:69] worker 7 listening for tasks. I0424 19:03:06.572688 1 tasks_processing.go:71] worker 6 working on validating_webhook_configurations task. I0424 19:03:06.572728 1 tasks_processing.go:71] worker 59 working on jaegers task. I0424 19:03:06.572738 1 tasks_processing.go:71] worker 41 working on crds task. I0424 19:03:06.572680 1 tasks_processing.go:71] worker 20 working on sap_datahubs task. I0424 19:03:06.572682 1 tasks_processing.go:69] worker 12 listening for tasks. I0424 19:03:06.572814 1 tasks_processing.go:71] worker 54 working on ceph_cluster task. I0424 19:03:06.572852 1 tasks_processing.go:71] worker 62 working on nodes task. I0424 19:03:06.572690 1 tasks_processing.go:71] worker 24 working on support_secret task. I0424 19:03:06.572265 1 tasks_processing.go:71] worker 36 working on container_images task. I0424 19:03:06.573011 1 tasks_processing.go:71] worker 47 working on olm_operators task. I0424 19:03:06.571644 1 tasks_processing.go:69] worker 35 listening for tasks. I0424 19:03:06.573227 1 tasks_processing.go:71] worker 26 working on nodenetworkstates task. I0424 19:03:06.573302 1 tasks_processing.go:71] worker 35 working on lokistack task. I0424 19:03:06.573539 1 tasks_processing.go:71] worker 13 working on openshift_logging task. I0424 19:03:06.573584 1 tasks_processing.go:71] worker 61 working on proxies task. I0424 19:03:06.573630 1 tasks_processing.go:71] worker 7 working on image task. I0424 19:03:06.573611 1 tasks_processing.go:71] worker 28 working on silenced_alerts task. I0424 19:03:06.573629 1 tasks_processing.go:71] worker 52 working on active_alerts task. I0424 19:03:06.573621 1 tasks_processing.go:71] worker 56 working on image_registries task. I0424 19:03:06.573641 1 tasks_processing.go:71] worker 48 working on schedulers task. I0424 19:03:06.573647 1 tasks_processing.go:71] worker 45 working on machine_autoscalers task. I0424 19:03:06.573655 1 tasks_processing.go:71] worker 53 working on ingress_certificates task. I0424 19:03:06.572695 1 tasks_processing.go:71] worker 10 working on networks task. I0424 19:03:06.573658 1 tasks_processing.go:71] worker 51 working on cluster_apiserver task. I0424 19:03:06.573663 1 tasks_processing.go:71] worker 23 working on operators task. I0424 19:03:06.573665 1 tasks_processing.go:71] worker 12 working on aggregated_monitoring_cr_names task. I0424 19:03:06.573616 1 tasks_processing.go:71] worker 25 working on machine_config_pools task. E0424 19:03:06.577163 1 gather_node_features.go:86] GatherNodeFeatures: NodeFeatures resource not found in openshift-nfd namespace (may not be installed) I0424 19:03:06.577175 1 tasks_processing.go:71] worker 17 working on pod_network_connectivity_checks task. I0424 19:03:06.577238 1 gather.go:177] gatherer "clusterconfig" function "node_features" took 5.46929ms to process 0 records I0424 19:03:06.577259 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 5.434482ms to process 0 records I0424 19:03:06.577270 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 5.283821ms to process 0 records I0424 19:03:06.577276 1 tasks_processing.go:71] worker 18 working on image_pruners task. I0424 19:03:06.577285 1 tasks_processing.go:74] worker 1 stopped. I0424 19:03:06.577294 1 tasks_processing.go:74] worker 16 stopped. I0424 19:03:06.577278 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 5.587535ms to process 0 records I0424 19:03:06.577339 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 5.572602ms to process 0 records I0424 19:03:06.577353 1 tasks_processing.go:74] worker 27 stopped. I0424 19:03:06.577376 1 tasks_processing.go:74] worker 8 stopped. E0424 19:03:06.577389 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0424 19:03:06.577398 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 5.435018ms to process 0 records I0424 19:03:06.578094 1 tasks_processing.go:74] worker 34 stopped. I0424 19:03:06.578106 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 5.764432ms to process 0 records E0424 19:03:06.578426 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.78.237:9091: connect: connection refused E0424 19:03:06.578427 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.78.237:9091: connect: connection refused I0424 19:03:06.578442 1 tasks_processing.go:74] worker 14 stopped. E0424 19:03:06.578448 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.78.237:9091: connect: connection refused I0424 19:03:06.578457 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 5.879921ms to process 0 records E0424 19:03:06.578465 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.78.237:9091: connect: connection refused I0424 19:03:06.578480 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 6.395097ms to process 0 records I0424 19:03:06.578491 1 tasks_processing.go:74] worker 60 stopped. I0424 19:03:06.579914 1 tasks_processing.go:74] worker 0 stopped. I0424 19:03:06.580458 1 recorder.go:75] Recording config/infrastructure with fingerprint=a3840191d267a63b565f8563173abd24596d0e34ab7acf4d6958c5c23641b4ee I0424 19:03:06.580477 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 8.196837ms to process 1 records I0424 19:03:06.580492 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 8.594615ms to process 0 records I0424 19:03:06.580504 1 tasks_processing.go:74] worker 42 stopped. I0424 19:03:06.580658 1 tasks_processing.go:74] worker 32 stopped. I0424 19:03:06.580854 1 controller.go:129] Initializing last reported time to 0001-01-01T00:00:00Z I0424 19:03:06.580870 1 controller.go:368] The initial operator extension status is healthy I0424 19:03:06.580878 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0424 19:03:06.580883 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0424 19:03:06.580886 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0424 19:03:06.580892 1 recorder.go:75] Recording config/featuregate with fingerprint=142cec67185881c7f0c4ed9a2ffa51d64285f2137f2b77b5e14b08cccb60b679 I0424 19:03:06.580901 1 controller.go:531] The operator is still being initialized I0424 19:03:06.580907 1 controller.go:554] The operator is healthy I0424 19:03:06.580911 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 9.003685ms to process 1 records I0424 19:03:06.580973 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0424 19:03:06.581041 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0424 19:03:06.581164 1 operator.go:328] started I0424 19:03:06.581183 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0424 19:03:06.582159 1 tasks_processing.go:74] worker 57 stopped. I0424 19:03:06.582173 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 9.97555ms to process 0 records I0424 19:03:06.582189 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 9.354429ms to process 0 records I0424 19:03:06.582197 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 8.944657ms to process 0 records I0424 19:03:06.582204 1 tasks_processing.go:74] worker 26 stopped. I0424 19:03:06.582208 1 tasks_processing.go:74] worker 54 stopped. I0424 19:03:06.582943 1 tasks_processing.go:74] worker 20 stopped. I0424 19:03:06.582955 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 10.161183ms to process 0 records I0424 19:03:06.583390 1 tasks_processing.go:74] worker 31 stopped. I0424 19:03:06.583400 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 11.749153ms to process 0 records I0424 19:03:06.583449 1 tasks_processing.go:74] worker 13 stopped. I0424 19:03:06.583461 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 9.857887ms to process 0 records I0424 19:03:06.583627 1 tasks_processing.go:74] worker 63 stopped. I0424 19:03:06.583766 1 recorder.go:75] Recording config/ingress with fingerprint=1744e6d4c89d2779ff667fd80dedade5bb3c6d80aed8505e30e717867e8335a9 I0424 19:03:06.583778 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 11.343854ms to process 1 records E0424 19:03:06.583791 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0424 19:03:06.583798 1 gather.go:177] gatherer "clusterconfig" function "machines" took 11.202491ms to process 0 records I0424 19:03:06.583803 1 tasks_processing.go:74] worker 33 stopped. I0424 19:03:06.583865 1 tasks_processing.go:74] worker 55 stopped. I0424 19:03:06.583939 1 recorder.go:75] Recording config/authentication with fingerprint=eeb510b5ac0b5ddea237502252cb200fb45db1ec4873513ccb22368d07d08145 I0424 19:03:06.583947 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 11.31395ms to process 1 records I0424 19:03:06.584890 1 tasks_processing.go:74] worker 39 stopped. I0424 19:03:06.584903 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 12.746424ms to process 0 records I0424 19:03:06.585149 1 tasks_processing.go:74] worker 21 stopped. I0424 19:03:06.585358 1 recorder.go:75] Recording config/oauth with fingerprint=37027745251c973aea7d54ff6b2d85a2473bb3444e5931a62a6f46270e1a6b96 I0424 19:03:06.585371 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 13.01306ms to process 1 records I0424 19:03:06.585764 1 tasks_processing.go:74] worker 47 stopped. I0424 19:03:06.585777 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 12.734769ms to process 0 records I0424 19:03:06.587278 1 tasks_processing.go:74] worker 30 stopped. I0424 19:03:06.587450 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=fc6a18d2f3d265fafb74149e2b4307db66d57c0e26ab3265697226f0ed7eac27 I0424 19:03:06.587481 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=1db762eed3068537521b0da3ed6fa42cfabbec97e2abc578a91405e953ce0439 I0424 19:03:06.587507 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=8237fb1988b018f4fc52530ceee3944cad003e01271084c3fe6f5ba8f87da412 I0424 19:03:06.587516 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 15.313232ms to process 3 records I0424 19:03:06.595377 1 tasks_processing.go:74] worker 35 stopped. I0424 19:03:06.595405 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 22.047366ms to process 0 records I0424 19:03:06.595433 1 tasks_processing.go:74] worker 59 stopped. I0424 19:03:06.595450 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 22.689507ms to process 0 records I0424 19:03:06.595469 1 tasks_processing.go:74] worker 2 stopped. I0424 19:03:06.595480 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 23.56835ms to process 0 records I0424 19:03:06.595576 1 tasks_processing.go:74] worker 61 stopped. I0424 19:03:06.595687 1 recorder.go:75] Recording config/proxy with fingerprint=4c4d0419cc3d911cb2736ace21de2ccebd6ef90be91a57e419d21f966b9d9065 I0424 19:03:06.595709 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 21.93571ms to process 1 records I0424 19:03:06.595776 1 tasks_processing.go:74] worker 3 stopped. I0424 19:03:06.595796 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0424 19:03:06.595806 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 23.876295ms to process 1 records I0424 19:03:06.595980 1 tasks_processing.go:74] worker 17 stopped. E0424 19:03:06.595990 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0424 19:03:06.595997 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 18.794947ms to process 0 records I0424 19:03:06.596119 1 tasks_processing.go:74] worker 50 stopped. I0424 19:03:06.596134 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 24.030295ms to process 0 records I0424 19:03:06.596141 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 24.022154ms to process 0 records I0424 19:03:06.596146 1 tasks_processing.go:74] worker 37 stopped. I0424 19:03:06.597487 1 tasks_processing.go:74] worker 45 stopped. I0424 19:03:06.597506 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 23.493814ms to process 0 records I0424 19:03:06.597517 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 23.251358ms to process 0 records I0424 19:03:06.597524 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 24.816214ms to process 0 records I0424 19:03:06.597528 1 tasks_processing.go:74] worker 25 stopped. I0424 19:03:06.597531 1 tasks_processing.go:74] worker 11 stopped. I0424 19:03:06.597592 1 tasks_processing.go:74] worker 15 stopped. I0424 19:03:06.597733 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=62fba03ea9a28a95210eba706dd3bd49ceaf8c4f9f778dd718b9e2e83ffa3901 I0424 19:03:06.597755 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=fc983ea3d97aa1e22c2466641d8f4a7b5ae6c0ab1424f25749ea2a7a8ea23453 I0424 19:03:06.597762 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 25.418032ms to process 2 records W0424 19:03:06.597842 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0424 19:03:06.599653 1 tasks_processing.go:74] worker 29 stopped. I0424 19:03:06.600123 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=592ef9f801503edd72f940b378b916bbef862736933f961ff3dd7a69828c174c I0424 19:03:06.601322 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=229422cbe40a2d6e0dfd5fff6ca1c3e7f72ea3109fa2b782a20999f50e6c2544 I0424 19:03:06.601461 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 27.985555ms to process 2 records I0424 19:03:06.601773 1 recorder.go:75] Recording config/image with fingerprint=9f5f433d6f1439aa00ebeda7155cf08b6094f1baba2ea41ec92cd806503678dd I0424 19:03:06.601867 1 gather.go:177] gatherer "clusterconfig" function "image" took 26.114795ms to process 1 records I0424 19:03:06.601929 1 tasks_processing.go:74] worker 7 stopped. I0424 19:03:06.602102 1 tasks_processing.go:74] worker 10 stopped. I0424 19:03:06.602620 1 recorder.go:75] Recording config/network with fingerprint=159b6e97506e72be05ee578a4a1703a07254fba53179dbfd3f6d804a3ea37710 I0424 19:03:06.602697 1 gather.go:177] gatherer "clusterconfig" function "networks" took 26.95534ms to process 1 records I0424 19:03:06.602773 1 gather_logs.go:145] no pods in namespace were found I0424 19:03:06.602808 1 tasks_processing.go:74] worker 46 stopped. I0424 19:03:06.602823 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 30.678588ms to process 0 records I0424 19:03:06.604875 1 tasks_processing.go:74] worker 18 stopped. I0424 19:03:06.605209 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=cccdc89b19b25d4cab681274ec29da6f6e57af1a62280d1ca2fc3ec6cc18405d I0424 19:03:06.605239 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 27.586234ms to process 1 records I0424 19:03:06.605562 1 tasks_processing.go:74] worker 24 stopped. E0424 19:03:06.605577 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0424 19:03:06.605585 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 32.581741ms to process 0 records I0424 19:03:06.605844 1 tasks_processing.go:74] worker 48 stopped. I0424 19:03:06.605922 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=13848d14da0e833d6bc26835db9e66525903552a05ae1e7d7919b62eb9e986e0 I0424 19:03:06.605937 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 31.959024ms to process 1 records I0424 19:03:06.605952 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 33.217144ms to process 0 records I0424 19:03:06.605976 1 tasks_processing.go:74] worker 38 stopped. I0424 19:03:06.606052 1 tasks_processing.go:74] worker 49 stopped. I0424 19:03:06.606129 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=dec5379c4d6724a1b0b41d3df2ba8194a4dd62ef59057e68680abefa44bbba0c I0424 19:03:06.606158 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=466589db80c397624bea17ea9731f73be6d25387faa95eb568fbf1c12d55d922 I0424 19:03:06.606180 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=c4f30761ea9d1a4448d12aebbae04439b711f881e124bb257ca29ad3ca875631 I0424 19:03:06.606203 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=32bf886bcd0f512020658e3433eb854ef623cfed4006b8470f416e55e0570fd4 I0424 19:03:06.606231 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=b85ec1158a82f475f53d758e54a9dd5f5c0bb98263e108da47682c0325a18448 I0424 19:03:06.606260 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=51b66ba608eab94120b09782fa5668b162a244c2a33eac37b0d50bad46c147f5 I0424 19:03:06.606288 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=97cda8b884397471d86ac8cdff50114f1d6953cb0ac10d3f87456f0409cc44d1 I0424 19:03:06.606319 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=6a06509772e6c497cdec265d23efaede99cf9ac7f4a48d158aeea527d54d7a91 I0424 19:03:06.606366 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=e988da9614a8aae980fab01978c6e38407a35e1b62c959bec929273f1a9c7773 I0424 19:03:06.606393 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=e1d3f3ef283945d81873dfe5994c3a781bc932fb7e05257460bc5280de263ef7 I0424 19:03:06.606417 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=83bc26775f9277038d0dbc215d359fd7d12768f697d5d454acdfb3d75a2c9d57 I0424 19:03:06.606442 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 33.24573ms to process 11 records I0424 19:03:06.606558 1 tasks_processing.go:74] worker 6 stopped. I0424 19:03:06.606582 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=de6d411b2f1b25cf8306a4c01402dcdeb2dae76228fb783678b53016367f4c7a I0424 19:03:06.606646 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=510a66637956184a88c394e9532e964789be1882928817c28c634aed775c0fe1 I0424 19:03:06.606743 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=847c49e1e6d01d09e55cecb921b9335a4308eb53b784f73b9fc1c9a3464777b3 I0424 19:03:06.606795 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=cfa18c6f385a85c411c1a3a73a140b0c544160a081f2ded90c114351c58c13c0 I0424 19:03:06.606836 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=411b9454dc765f776a43eff92df9bf19ba36ff7c2a278dc2bbdd0665dc93222e I0424 19:03:06.606870 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=116861a25f06f95d71d8f869316b5d373b065d9f7f785193a8422b1f77603a56 I0424 19:03:06.606917 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=4c0d9bddf4551e0087fb5404baa63390f10b50bd5c3961b7c023d7903a137ed7 I0424 19:03:06.606962 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=eea9a5f7811000924946bfeccb6e8e2898c17e0b5b0b122dbc88e4c258250f48 I0424 19:03:06.607005 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=0fd4b2ef56a3c3da7a65a9e4599dfd85663e47aeb78bd3b609b75cf046bdd092 I0424 19:03:06.607048 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=a272d346ad68daf7159c103f1c7b106d6c5efb015e3112b22533e8b5e10eac72 I0424 19:03:06.607081 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=95c923ae6ddf5cdfe77a5c71d9bf0a9949c540cd4db49b2748d09ba731dce94a I0424 19:03:06.607097 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0424 19:03:06.607117 1 controller.go:254] Source scaController *sca.Controller is not ready I0424 19:03:06.607120 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=8999a930f3b550ba13b280ae369decd1b66aa8665b252a7bfee3941cba5f53f2 I0424 19:03:06.607125 1 controller.go:254] Source clusterTransferController *clustertransfer.Controller is not ready I0424 19:03:06.607131 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0424 19:03:06.607135 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0424 19:03:06.607157 1 controller.go:531] The operator is still being initialized I0424 19:03:06.607171 1 controller.go:554] The operator is healthy I0424 19:03:06.607177 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=f7275967129bd3b38c5237575f5d62b149cc929bd3891eef05bb89faef18a17d I0424 19:03:06.607188 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 33.373786ms to process 13 records I0424 19:03:06.607203 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 34.250989ms to process 0 records I0424 19:03:06.607250 1 tasks_processing.go:74] worker 4 stopped. I0424 19:03:06.607305 1 tasks_processing.go:74] worker 51 stopped. I0424 19:03:06.607661 1 recorder.go:75] Recording config/apiserver with fingerprint=d1b067e69c797693d25f5f114cf69edbdd159b4fe221452917e265d9927d3717 I0424 19:03:06.607679 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 32.646472ms to process 1 records I0424 19:03:06.607708 1 tasks_processing.go:74] worker 44 stopped. E0424 19:03:06.607688 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0424 19:03:06.607809 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0424 19:03:06.607827 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 34.588517ms to process 1 records I0424 19:03:06.607919 1 tasks_processing.go:74] worker 62 stopped. I0424 19:03:06.608527 1 recorder.go:75] Recording config/node/ip-10-0-0-68.ec2.internal with fingerprint=cc7c46d9e90dc283291328bfa16a68527fe5a65a9bd20408691751c3e655d728 I0424 19:03:06.608653 1 recorder.go:75] Recording config/node/ip-10-0-1-34.ec2.internal with fingerprint=7d08ece77161eaf35b49e5380cab69271683f866a12b861cedba5d93ed23f855 I0424 19:03:06.608830 1 recorder.go:75] Recording config/node/ip-10-0-2-233.ec2.internal with fingerprint=d43b2ae69e74e164efe633f1e117898c9839f42c121f2aa902b386b838438b37 I0424 19:03:06.608883 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 34.024076ms to process 3 records I0424 19:03:06.609025 1 tasks_processing.go:74] worker 56 stopped. I0424 19:03:06.610356 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=efd5e45b640abc705d68bf2cd4409d420e2674850c67f926091563c3482b8fc0 I0424 19:03:06.610383 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 33.125443ms to process 1 records I0424 19:03:06.610397 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 36.345197ms to process 0 records I0424 19:03:06.610406 1 tasks_processing.go:74] worker 58 stopped. I0424 19:03:06.611726 1 tasks_processing.go:74] worker 19 stopped. I0424 19:03:06.612116 1 recorder.go:75] Recording config/version with fingerprint=f2fc232a5c3b6011e4d867da168446d44d47310ed8e91acf55e12dc39e7b2e47 I0424 19:03:06.612143 1 recorder.go:75] Recording config/id with fingerprint=9e91762f108fc8b898594d4dfdde407d4bedfcc414d4202dac65d74793490800 I0424 19:03:06.612159 1 gather.go:177] gatherer "clusterconfig" function "version" took 39.131723ms to process 2 records I0424 19:03:06.618411 1 tasks_processing.go:74] worker 41 stopped. I0424 19:03:06.619094 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=88652c0fdc7a9cd667e9ea169e86a400aa311385bdaefd40c272535042e12c23 I0424 19:03:06.619419 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=e2844f2ee51b7aedabe57837bcb92405de5a0d0020c0c8b91a6a7c4776723f15 I0424 19:03:06.619445 1 gather.go:177] gatherer "clusterconfig" function "crds" took 45.661037ms to process 2 records I0424 19:03:06.619459 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 44.711116ms to process 0 records I0424 19:03:06.619468 1 tasks_processing.go:74] worker 12 stopped. I0424 19:03:06.625765 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0424 19:03:06.631951 1 tasks_processing.go:74] worker 36 stopped. I0424 19:03:06.634013 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-66dbbb6468-f9swk with fingerprint=7e2b4980b9f3a0d90956736a9d19967e3fc642c6cd74166f9c429f0fd9f92d39 I0424 19:03:06.634153 1 recorder.go:75] Recording config/running_containers with fingerprint=80e8cfee3bad9355ba8245092d21cfd21f404b63d8b411ee4dbccad9999142ad I0424 19:03:06.634169 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 58.928228ms to process 2 records I0424 19:03:06.636352 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0424 19:03:06.636363 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0424 19:03:06.636415 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0424 19:03:06.637090 1 tasks_processing.go:74] worker 22 stopped. E0424 19:03:06.637104 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0424 19:03:06.637111 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0424 19:03:06.637115 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0424 19:03:06.637124 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=ad286723f58bdcfc37aeba3ec5b4110c08e1af59cd34d14b4bfaab02d18e4856 I0424 19:03:06.637159 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0424 19:03:06.637166 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0424 19:03:06.637186 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=2ab5ab7b1b10d7fcf1197bb24dea7c90f400e4effc18c7356873209d54fdf84b I0424 19:03:06.637199 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0424 19:03:06.637264 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0424 19:03:06.637272 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0424 19:03:06.637277 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 65.038101ms to process 7 records I0424 19:03:06.637363 1 tasks_processing.go:74] worker 9 stopped. I0424 19:03:06.637380 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 65.202313ms to process 0 records I0424 19:03:06.642892 1 base_controller.go:82] Caches are synced for ConfigController I0424 19:03:06.642903 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0424 19:03:06.648475 1 configmapobserver.go:84] configmaps "insights-config" not found I0424 19:03:06.682218 1 base_controller.go:82] Caches are synced for LoggingSyncer I0424 19:03:06.682236 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0424 19:03:06.692784 1 cluster_transfer.go:102] no available accepted cluster transfer I0424 19:03:06.692802 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0424 19:03:06.718504 1 tasks_processing.go:74] worker 53 stopped. E0424 19:03:06.718525 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0424 19:03:06.718531 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2psmom7g8g3evconfugfskclt409077f-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2psmom7g8g3evconfugfskclt409077f-primary-cert-bundle-secret" not found I0424 19:03:06.718614 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=701257df3f20d4f0e37f925a6634926656d361cf66730e409769e3232a454be5 I0424 19:03:06.718629 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 144.455789ms to process 1 records I0424 19:03:06.811847 1 sca.go:266] etc-pki-entitlement secret successfully created I0424 19:03:06.811868 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated W0424 19:03:07.597410 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0424 19:03:07.655061 1 tasks_processing.go:74] worker 28 stopped. I0424 19:03:07.655085 1 recorder.go:75] Recording config/silenced_alerts.json with fingerprint=37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570 I0424 19:03:07.655098 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 1.081266687s to process 1 records I0424 19:03:07.655210 1 tasks_processing.go:74] worker 52 stopped. I0424 19:03:07.655250 1 recorder.go:75] Recording config/alerts with fingerprint=4a837865d0e4f8a48833685444fd64d06e066193db4aee312deda794c224a416 I0424 19:03:07.655263 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 1.081361918s to process 1 records W0424 19:03:08.598346 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0424 19:03:09.033632 1 tasks_processing.go:74] worker 23 stopped. I0424 19:03:09.033772 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=f6233d50d70b49ef94ef876f7583bb02e511e11bcba1f20f927c79b780437be6 I0424 19:03:09.033816 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0424 19:03:09.033851 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=1c5257e04a393c6fc69408d960638a65bc15e5e6b43e7c142a544aaa207a939e I0424 19:03:09.033860 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0424 19:03:09.033882 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=e7607e3470f5bad28ccf36c559f74575424f591d13dc1b87c674eb4337b7155e I0424 19:03:09.033897 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0424 19:03:09.033923 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=743f0def3af899c08586c2298395663758136dee1ff01ac24c5b2f9ebfb8c3b7 I0424 19:03:09.033950 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=58eb5d7418f7d28af6acb6abce93daea5589f2d4672eadecde40bb9ee75346b8 I0424 19:03:09.033972 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=bf85cb0f414fae1a56a7101b14967afae99c810508bda871ccf3b169f8a58e75 I0424 19:03:09.034001 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=f07ca4044181e383883b511ef10e7cf335a6f3b97aee08a33756695bb94a9ebd I0424 19:03:09.034247 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=26f895558bf639da45ce4daf63768e755b50c1271ddc5426656ed813d2d25f07 I0424 19:03:09.034265 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=18d1b7140a4bc0bfbb4cb90e530cc83c835eb44e42643b903228721b444cc32c I0424 19:03:09.034272 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0424 19:03:09.034291 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=34e73dfe513f9a9e8fa22f5d11338da7fd3c5e8b8a969ec1e8cb269c23e49a7c I0424 19:03:09.034301 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0424 19:03:09.034337 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=3dbf2ffad38a00c1dc4c15240a7872b91d69b13a9ebce51732f7e904a4d9c6a7 I0424 19:03:09.034348 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0424 19:03:09.034367 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=a3601b6609727e84a5c7a10bd5e067f3c33d66ac4b1f1170a33270e1e1493561 I0424 19:03:09.034377 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0424 19:03:09.034395 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=50577bc98bb19b27f0a10f3accf05c0182346b645f0bb05d2f33e19845bc0212 I0424 19:03:09.034498 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=79a78e13ab14fc4df22bd6b7a45e5ee0afa98420bb727a239bd4fbab00108d0f I0424 19:03:09.034507 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0424 19:03:09.034514 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0424 19:03:09.034535 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0424 19:03:09.034560 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=954e8f70731eee6f067949fc7ede82a1a4758fc691ad35b05ec1214ee1031be6 I0424 19:03:09.034582 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=e122402cae506160a6f60fb5eba5f3f8ca38fc6f91d97e2a91d0b2039962c078 I0424 19:03:09.034591 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0424 19:03:09.034606 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=99b5a7c9b203b63d359046147db0cc87041ada006e268d48b7e87aad504fec9e I0424 19:03:09.034614 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0424 19:03:09.034630 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=16f34658a2391c1c087bd9c2a67b07253afe21aa066dc65cbcf720ff11be9df6 I0424 19:03:09.034639 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0424 19:03:09.034652 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=1ca4da6b327700c619de39d9e6180162ea9ff028b151d0aa08d3461e72b28a05 I0424 19:03:09.034667 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=4b7025729250762b682849e32ff6a15762aa5f3bee7e6f4fa8fc7e3c1267c602 I0424 19:03:09.034680 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=948fcdf184ba960edd7ee545359a106314b2fcc0138298b7b26aa3801355e3c0 I0424 19:03:09.034701 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=4310cee7bc6acd6958684e0860d6e167e108c7646cef47b5ff3238cd939372bc I0424 19:03:09.034711 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0424 19:03:09.034739 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=a4d7fbd97124471666d4f0951258c2ff46ee6f05031d72f51346a4b04814cd58 I0424 19:03:09.034755 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0424 19:03:09.034764 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0424 19:03:09.034771 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.459449686s to process 39 records W0424 19:03:09.598440 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0424 19:03:10.597674 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0424 19:03:11.598342 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0424 19:03:11.598369 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0424 19:03:11.598385 1 tasks_processing.go:74] worker 43 stopped. E0424 19:03:11.598397 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0424 19:03:11.598410 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0424 19:03:11.598421 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.026197635s to process 1 records I0424 19:03:19.410521 1 tasks_processing.go:74] worker 5 stopped. I0424 19:03:19.410565 1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b I0424 19:03:19.410581 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.838788587s to process 1 records I0424 19:03:20.179434 1 tasks_processing.go:74] worker 40 stopped. I0424 19:03:20.179702 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=80e5b054934e37ddf232a6b52185c88ef3149965af63f70b0125fd661e8f707e I0424 19:03:20.179720 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.60695923s to process 1 records E0424 19:03:20.179797 1 periodic.go:254] "Unhandled Error" err="clusterconfig failed after 13.608s with: function \"machine_healthchecks\" failed with an error, function \"tsdb_status\" failed with an error, function \"metrics\" failed with an error, function \"machines\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"support_secret\" failed with an error, function \"machine_configs\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0424 19:03:20.180911 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "machine_healthchecks" failed with an error, function "tsdb_status" failed with an error, function "metrics" failed with an error, function "machines" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "support_secret" failed with an error, function "machine_configs" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0424 19:03:20.180928 1 periodic.go:216] Running workloads gatherer I0424 19:03:20.180943 1 tasks_processing.go:45] number of workers: 2 I0424 19:03:20.180949 1 tasks_processing.go:69] worker 1 listening for tasks. I0424 19:03:20.180953 1 tasks_processing.go:71] worker 1 working on workload_info task. I0424 19:03:20.180959 1 tasks_processing.go:69] worker 0 listening for tasks. I0424 19:03:20.180981 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0424 19:03:20.188137 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-68.ec2.internal... I0424 19:03:20.188154 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-34.ec2.internal... I0424 19:03:20.188174 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-233.ec2.internal... I0424 19:03:20.210872 1 tasks_processing.go:74] worker 0 stopped. I0424 19:03:20.210898 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 29.873454ms to process 0 records I0424 19:03:24.599687 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 4s I0424 19:03:24.626408 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0424 19:03:24.653357 1 gather_workloads_info.go:387] No image sha256:a043239802b3eb8b323d285193d2527fad0ecec98ca91d188a3472a2fac8ae04 (28ms) I0424 19:03:24.662704 1 gather_workloads_info.go:387] No image sha256:730d1b6988025bef0daa3a9a5d8467ec4a26b0382cc52f91c3375b4590d3518a (9ms) I0424 19:03:24.672050 1 gather_workloads_info.go:387] No image sha256:0acfe677ed58031f8381cb8a4f2d8917ec4cfa95485e2a46529dffae54c75ace (9ms) I0424 19:03:24.681594 1 gather_workloads_info.go:387] No image sha256:48f5559439bea030a6423652da448c25fff0c62ba293eb521843c1a8eb89ddb0 (10ms) I0424 19:03:24.690586 1 gather_workloads_info.go:387] No image sha256:5ff204630794311b0b37fa7b197b933ad85d76a481bf7cdb3bcbada08f0cdcbf (9ms) I0424 19:03:24.699968 1 gather_workloads_info.go:387] No image sha256:5d6a2b8d2508c7da71192661e7c0dcf1335f48b6336a20189abffbe79eeff962 (9ms) I0424 19:03:24.708781 1 gather_workloads_info.go:387] No image sha256:ab60623bb32f7e75fca71ef65137731cae347a21c7a4091dfd583fa00732721c (9ms) I0424 19:03:24.717788 1 gather_workloads_info.go:387] No image sha256:25c148fd380b1a9db3f6039d2e0eabc489a954921452391390ba9192b2325678 (9ms) I0424 19:03:24.726339 1 gather_workloads_info.go:387] No image sha256:934d8e8c50f3c609b8eea80d1051111fe3d066fe8c65c79572072ae55fcb0a86 (9ms) I0424 19:03:24.735572 1 gather_workloads_info.go:387] No image sha256:5b6212b8f539f08e78417d8a4b7485ca0b4e7927cacd7b752742a28841bc8ccd (9ms) I0424 19:03:24.744876 1 gather_workloads_info.go:387] No image sha256:0a4dfb8d4c1b3849319d45b4c54dff26a7238a2c08fcaa121f93073e95ab12e8 (9ms) I0424 19:03:24.836649 1 gather_workloads_info.go:398] Found image sha256:673ebc8cc22c56c8f410e011b2fa950d28cf7b6420e17fdb6580d6cb10523384 (92ms) I0424 19:03:24.936653 1 gather_workloads_info.go:387] No image sha256:ae7d3453fd734ecc865e5f9bb16f29244ebffe6291b77e1d4e496f71eb053174 (100ms) I0424 19:03:25.035587 1 gather_workloads_info.go:387] No image sha256:d64bea34bf3e1bb0b3a701c3ff14e66665afc1b050f28124ad7e6888eaec3a81 (99ms) I0424 19:03:25.135872 1 gather_workloads_info.go:387] No image sha256:572ebf7c7fab1a518582f3ca46c866730423cce6e8a19f614dc15fd61bf0c65a (100ms) I0424 19:03:25.235718 1 gather_workloads_info.go:387] No image sha256:b958e39c1643d98648b20ff8b6ec38a51c9c6606f2eecff4044d41ac993e1878 (100ms) I0424 19:03:25.336368 1 gather_workloads_info.go:387] No image sha256:5ac9c549d65fc1d8bc900773bebee43e9192bcec1bb5fa46afb4597230c16ac7 (101ms) I0424 19:03:25.435887 1 gather_workloads_info.go:387] No image sha256:48e883a3932aea9457f55cd4628d21397d429d4828f42f95d1c903d9d5395bde (99ms) I0424 19:03:25.535912 1 gather_workloads_info.go:387] No image sha256:3bba1358d4a0ae878ff491c0c2cbfffe60649e110b40342b878fe8fa332f8858 (100ms) I0424 19:03:25.635004 1 gather_workloads_info.go:387] No image sha256:3380fecebaeee139c1607fbcd9346cebc92806679568e2244a8905ef0f52aad2 (99ms) I0424 19:03:25.735076 1 gather_workloads_info.go:387] No image sha256:0092742d8c2f3e9241f8f87d6f1da2a063627c140c491dc9c5e97526f760101a (100ms) I0424 19:03:25.835928 1 gather_workloads_info.go:387] No image sha256:6814f5fced0ef219d06374011c68a11a3da788a764a00a69ade435466d9ee240 (101ms) I0424 19:03:25.936146 1 gather_workloads_info.go:387] No image sha256:17aefdc2f4752c23dc8f0947bb01fc34635822eeb18f2d8a5eddf8211b84d450 (100ms) I0424 19:03:26.036143 1 gather_workloads_info.go:387] No image sha256:084aa9b0f8a6d478549dc384d4e66da13ee9b25cc98531da861cc19dee2a9e8f (100ms) I0424 19:03:26.136407 1 gather_workloads_info.go:387] No image sha256:000105ef5150e7079b90a613fb9e6193e2a6ef9b1908d2dce44f2395d4fd070f (100ms) I0424 19:03:26.235713 1 gather_workloads_info.go:387] No image sha256:6c6d412ccf0c3d6a14645a79df6c4658c0f8725d7954cb83c3d2f0116499345a (99ms) I0424 19:03:26.335979 1 gather_workloads_info.go:387] No image sha256:bdeb1c455cffccdb6c90db238e9d371de06340b1d6105d47f66d87293f878ac1 (100ms) I0424 19:03:26.434942 1 gather_workloads_info.go:387] No image sha256:55b1db6038c5beaed54c626e3343b7a8589cc0be8dc41d1a66b4deab766ff520 (99ms) I0424 19:03:26.535907 1 gather_workloads_info.go:387] No image sha256:08c5a78c8a5af04c549e2273aaf4bb452a75bf038d68aa9d01bb2aff66c30e90 (101ms) I0424 19:03:26.635639 1 gather_workloads_info.go:387] No image sha256:15e1b5f763553e86d0230a751b0783e3b990438fd07cf96eed164bccb72ed564 (100ms) I0424 19:03:26.738579 1 gather_workloads_info.go:387] No image sha256:52db05084f2a2e7d7bf4040e4865df88ec4074efbbe59a78d861be2c5ab2aa40 (103ms) I0424 19:03:26.835653 1 gather_workloads_info.go:387] No image sha256:5940cc34b322325618929b1e0eb3864dae73d54c8f5b474e218dfb28123a61c7 (97ms) I0424 19:03:26.936092 1 gather_workloads_info.go:387] No image sha256:1eb6a6ee3c66ac15be6bc21bb6015c595c11c15b9dfd21bd9f5ce7f7b990ebaa (100ms) I0424 19:03:27.036222 1 gather_workloads_info.go:387] No image sha256:15677f0b70e6aa2dfaf088e45fc1a425c22bf6fda326b8116f87e88b6694dfab (100ms) I0424 19:03:27.136121 1 gather_workloads_info.go:387] No image sha256:5c6d21c3f97366bc7ab57031cc027b67405a684bd804ce364ed5998b0685eaca (100ms) I0424 19:03:27.235001 1 gather_workloads_info.go:387] No image sha256:0b2e6536a0ca3afc5e64a218087a6287c74b77a359075a385f1949ca5bd51b22 (99ms) I0424 19:03:27.335427 1 gather_workloads_info.go:387] No image sha256:89277d8d4560d71db88c2dcc67c992a24544ca21810920b609c1d49d53b4a287 (100ms) I0424 19:03:27.436054 1 gather_workloads_info.go:387] No image sha256:295fdda3bdb8bc5a12a5b038d087001f9e37481cf6f1b3a6d80d39dfabe17af9 (101ms) I0424 19:03:27.535559 1 gather_workloads_info.go:387] No image sha256:53ff8efdd8f563421ebc1859ab021a3bbc1306f82d4ff5d9c21bc60ec5f57898 (99ms) I0424 19:03:27.635984 1 gather_workloads_info.go:387] No image sha256:9d6d63dce784df5863e6a0ba4c11317a26c9d756d4e62d47759c97190a71cce5 (100ms) I0424 19:03:27.735260 1 gather_workloads_info.go:387] No image sha256:875c77e5d144f03fb91d8cee0259f6966683ca88d1bf818dbf4652c16b70312c (99ms) I0424 19:03:27.835932 1 gather_workloads_info.go:387] No image sha256:5aaea0419169e55832cc27acfe0fe3b9513a343d6bdf71d3da1575ed322245d0 (101ms) I0424 19:03:27.936375 1 gather_workloads_info.go:387] No image sha256:5808401268394502d335281ea1a294b07210461b986b58f91d7d1f29c0029c6d (100ms) I0424 19:03:28.035959 1 gather_workloads_info.go:387] No image sha256:9ef6dc311b880a6442b7a3a19006ddd955c7da36850bf000f5c9b68fd0862dd4 (100ms) I0424 19:03:28.136025 1 gather_workloads_info.go:398] Found image sha256:23204877f166455ce8f9de9b91a467eeda576f172d5f96124a68aa986c4775d6 (100ms) I0424 19:03:28.240512 1 gather_workloads_info.go:387] No image sha256:8f881f6f0498eaba860d711dc8e9dc066db8728f22abedb82e1dc9a8e1c30046 (104ms) I0424 19:03:28.335878 1 gather_workloads_info.go:387] No image sha256:30e597ec5d6bb96ff70a4f8688c748b659cd4fd5d73d222e8701821d236795c5 (95ms) I0424 19:03:28.436391 1 gather_workloads_info.go:387] No image sha256:15928a5b803fa00781b408dbc4c3bfa7c5287b18584ba31f817def8a4430c725 (100ms) I0424 19:03:28.535861 1 gather_workloads_info.go:387] No image sha256:637d41f067a5239096fc22b135181cda5113da833f1370e7a73965e83792e93a (99ms) I0424 19:03:28.535903 1 tasks_processing.go:74] worker 1 stopped. E0424 19:03:28.535916 1 gather.go:140] gatherer "workloads" function "workload_info" failed with the error: received unexpected status code 500 Internal Server Error from https://10.128.0.25:8443/gather_runtime_info I0424 19:03:28.536225 1 recorder.go:75] Recording config/workload_info with fingerprint=b2da08fc456960024d58f578f5c2c6cc242474f2f0dd4b6f663da9bf4fc005f2 I0424 19:03:28.536246 1 gather.go:177] gatherer "workloads" function "workload_info" took 8.354940004s to process 1 records E0424 19:03:28.536281 1 periodic.go:254] "Unhandled Error" err="workloads failed after 8.355s with: function \"workload_info\" failed with an error" I0424 19:03:28.537950 1 controllerstatus.go:89] name=periodic-workloads healthy=false reason=PeriodicGatherFailed message=Source workloads could not be retrieved: function "workload_info" failed with an error I0424 19:03:28.537965 1 periodic.go:216] Running conditional gatherer I0424 19:03:28.544778 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.21.9/gathering_rules I0424 19:03:28.724579 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer I0424 19:03:29.810440 1 conditional_gatherer.go:374] alert "ClusterMonitoringOperatorReconciliationErrors" has state "pending" I0424 19:03:29.810456 1 conditional_gatherer.go:374] alert "AlertmanagerReceiversNotConfigured" has state "pending" I0424 19:03:29.810539 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0424 19:03:29.817470 1 conditional_gatherer.go:392] cluster version is '4.21.9' I0424 19:03:29.817501 1 tasks_processing.go:45] number of workers: 3 I0424 19:03:29.817511 1 tasks_processing.go:69] worker 2 listening for tasks. I0424 19:03:29.817515 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0424 19:03:29.817520 1 tasks_processing.go:69] worker 0 listening for tasks. I0424 19:03:29.817531 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0424 19:03:29.817537 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0424 19:03:29.817538 1 tasks_processing.go:69] worker 1 listening for tasks. I0424 19:03:29.817546 1 tasks_processing.go:74] worker 1 stopped. I0424 19:03:29.817612 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=c8d4892cd4f2eea27d57353c6055eeb530bc2d811bfe041f6984d90395255a8c I0424 19:03:29.817626 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.019µs to process 1 records I0424 19:03:29.817659 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0424 19:03:29.817666 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.518µs to process 1 records I0424 19:03:29.817672 1 tasks_processing.go:74] worker 0 stopped. I0424 19:03:29.817773 1 tasks_processing.go:74] worker 2 stopped. I0424 19:03:29.817783 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 231.367µs to process 0 records I0424 19:03:29.817797 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0424 19:03:29.817812 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0424 19:03:29.847093 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=b2e6e548d1c8c9b06b7cebfa51347a4bc65ac43a4ac57ecfb3f81038a924f52c I0424 19:03:29.847236 1 diskrecorder.go:70] Writing 110 records to /var/lib/insights-operator/insights-2026-04-24-190329.tar.gz I0424 19:03:29.853381 1 diskrecorder.go:51] Wrote 110 records to disk in 6ms I0424 19:03:29.853417 1 periodic.go:285] Gathering cluster info every 2h0m0s I0424 19:03:29.853447 1 periodic.go:286] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0424 19:04:36.571784 1 diskrecorder.go:223] Found files to send: insights-2026-04-24-190329.tar.gz I0424 19:04:36.571815 1 insightsuploader.go:150] Checking archives to upload periodically every 16m1.076955073s I0424 19:04:36.571823 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0424 19:04:36.587089 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0424 19:04:36.825158 1 requests.go:87] Successfully reported id=2026-04-24T19:04:36Z x-rh-insights-request-id=f2db19cb801c4047a45e205784baeaca, wrote=48616 I0424 19:04:36.825258 1 insightsuploader.go:187] Uploaded report successfully in 253.431176ms I0424 19:04:36.825275 1 controller.go:129] Initializing last reported time to 2026-04-24T19:04:36Z I0424 19:04:36.825350 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0424 19:04:36.825366 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0424 19:04:36.825375 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0424 19:04:36.831469 1 controller.go:554] The operator is healthy I0424 19:05:06.596284 1 controller.go:554] The operator is healthy I0424 19:05:42.710529 1 insightsreport.go:137] Pulling report from smart-proxy I0424 19:05:42.710558 1 insightsreport.go:149] Retrieving report I0424 19:05:42.718184 1 requests.go:110] Retrieving report for cluster: 6409f3bb-de3f-4908-84bb-066f4a51a626 I0424 19:05:42.718197 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/6409f3bb-de3f-4908-84bb-066f4a51a626/reports I0424 19:05:42.721189 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/6409f3bb-de3f-4908-84bb-066f4a51a626/reports I0424 19:05:42.945168 1 insightsreport.go:184] Report retrieved I0424 19:05:42.976405 1 insightsreport.go:239] Report retrieved correctly I0424 19:07:06.591678 1 controller.go:554] The operator is healthy I0424 19:08:06.562990 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0424 19:08:06.580782 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0424 19:08:06.580802 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0424 19:08:06.586043 1 secretconfigobserver.go:119] support secret does not exist I0424 19:09:06.593813 1 controller.go:554] The operator is healthy I0424 19:11:06.588657 1 controller.go:554] The operator is healthy I0424 19:13:06.559230 1 configmapobserver.go:84] configmaps "insights-config" not found I0424 19:13:06.586511 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0424 19:13:06.588016 1 controller.go:554] The operator is healthy I0424 19:13:06.591965 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0424 19:13:06.591984 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0424 19:13:06.596840 1 secretconfigobserver.go:119] support secret does not exist I0424 19:15:06.590641 1 controller.go:554] The operator is healthy I0424 19:17:06.589658 1 controller.go:554] The operator is healthy I0424 19:18:06.597198 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0424 19:18:06.603241 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0424 19:18:06.603263 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0424 19:18:06.608028 1 secretconfigobserver.go:119] support secret does not exist I0424 19:19:06.588976 1 controller.go:554] The operator is healthy I0424 19:20:42.903080 1 insightsuploader.go:146] Nothing to report since 2026-04-24T19:04:36Z I0424 19:21:06.589044 1 controller.go:554] The operator is healthy I0424 19:23:06.574385 1 configmapobserver.go:84] configmaps "insights-config" not found I0424 19:23:06.596698 1 controller.go:554] The operator is healthy I0424 19:23:06.608553 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0424 19:23:06.615490 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0424 19:23:06.615509 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0424 19:23:06.623907 1 secretconfigobserver.go:119] support secret does not exist I0424 19:25:06.587987 1 controller.go:554] The operator is healthy I0424 19:27:06.589139 1 controller.go:554] The operator is healthy I0424 19:28:06.624938 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0424 19:28:06.630936 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0424 19:28:06.630956 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0424 19:28:06.636146 1 secretconfigobserver.go:119] support secret does not exist I0424 19:29:06.587870 1 controller.go:554] The operator is healthy I0424 19:31:06.590205 1 controller.go:554] The operator is healthy I0424 19:33:06.561974 1 configmapobserver.go:84] configmaps "insights-config" not found I0424 19:33:06.587497 1 controller.go:554] The operator is healthy I0424 19:33:06.636448 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0424 19:33:06.642197 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0424 19:33:06.642216 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0424 19:33:06.647476 1 secretconfigobserver.go:119] support secret does not exist I0424 19:35:06.588763 1 controller.go:554] The operator is healthy I0424 19:36:03.517471 1 insightsuploader.go:146] Nothing to report since 2026-04-24T19:04:36Z