I0507 15:33:03.171743 1 cmd.go:253] Using service-serving-cert provided certificates I0507 15:33:03.172189 1 observer_polling.go:159] Starting file observer I0507 15:33:03.634886 1 operator.go:76] Starting insights-operator v0.0.0-master+$Format:%H$ I0507 15:33:03.635077 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0507 15:33:03.635648 1 secure_serving.go:57] Forcing use of http/1.1 only W0507 15:33:03.635670 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0507 15:33:03.635674 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0507 15:33:03.635678 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0507 15:33:03.635681 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0507 15:33:03.635683 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0507 15:33:03.635686 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0507 15:33:03.635703 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0507 15:33:03.641439 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"17e067cc-3108-4f1c-9be4-29ff95f4b933", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GCPClusterHostedDNSInstall", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "HyperShiftOnlyDynamicResourceAllocation", "ImageStreamImportMode", "ImageVolume", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "PreconfiguredUDNAddresses", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SigstoreImageVerification", "SigstoreImageVerificationPKI", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks", "VolumeAttributesClass"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSDualStackInstall", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureDualStackInstall", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "CBORServingAndStorage", "CRDCompatibilityRequirementOperator", "ClientsAllowCBOR", "ClientsPreferCBOR", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterAPIMachineManagement", "ClusterAPIMachineManagementVSphere", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "EtcdBackendQuota", "EventTTL", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "GCPDualStackInstall", "ImageModeStatusReporting", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryption", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesCPMS", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutableCSINodeAllocatableCount", "MutatingAdmissionPolicy", "NewOLM", "NewOLMBoxCutterRuntime", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterInstall", "NutanixMultiSubnets", "OSStreams", "OVNObservability", "OnPremDNSRecords", "OpenShiftPodSecurityAdmission", "ProvisioningRequestAvailable", "SELinuxMount", "ShortCertRotation", "SignatureStores", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeGroupSnapshot"}} I0507 15:33:03.641383 1 operator.go:141] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GCPClusterHostedDNSInstall GatewayAPI GatewayAPIController HighlyAvailableArbiter HyperShiftOnlyDynamicResourceAllocation ImageStreamImportMode ImageVolume KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS ManagedBootImagesAzure ManagedBootImagesvSphere MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages PreconfiguredUDNAddresses ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SigstoreImageVerification SigstoreImageVerificationPKI StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks VolumeAttributesClass AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSDualStackInstall AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureDualStackInstall AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement CBORServingAndStorage CRDCompatibilityRequirementOperator ClientsAllowCBOR ClientsPreferCBOR ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterAPIMachineManagement ClusterAPIMachineManagementVSphere ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud EtcdBackendQuota EventTTL EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall GCPDualStackInstall ImageModeStatusReporting IngressControllerDynamicConfigurationManager InsightsConfig InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryption KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesCPMS MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutableCSINodeAllocatableCount MutatingAdmissionPolicy NewOLM NewOLMBoxCutterRuntime NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterInstall NutanixMultiSubnets OSStreams OVNObservability OnPremDNSRecords OpenShiftPodSecurityAdmission ProvisioningRequestAvailable SELinuxMount ShortCertRotation SignatureStores TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeGroupSnapshot] I0507 15:33:03.642576 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0507 15:33:03.642593 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0507 15:33:03.642595 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0507 15:33:03.642607 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0507 15:33:03.642612 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0507 15:33:03.642609 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0507 15:33:03.642897 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" I0507 15:33:03.643200 1 secure_serving.go:213] Serving securely on [::]:8443 I0507 15:33:03.643229 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0507 15:33:03.651602 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0507 15:33:03.651621 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0507 15:33:03.651737 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0507 15:33:03.658734 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0507 15:33:03.658751 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0507 15:33:03.664091 1 secretconfigobserver.go:119] support secret does not exist I0507 15:33:03.669293 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0507 15:33:03.674149 1 secretconfigobserver.go:119] support secret does not exist I0507 15:33:03.677703 1 recorder.go:176] Pruning old reports every 5h27m29s, max age is 288h0m0s I0507 15:33:03.683507 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0507 15:33:03.683510 1 periodic.go:216] Running clusterconfig gatherer I0507 15:33:03.683526 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0507 15:33:03.683562 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0507 15:33:03.683565 1 tasks_processing.go:45] number of workers: 64 I0507 15:33:03.683571 1 insightsreport.go:296] Starting report retriever I0507 15:33:03.683576 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0507 15:33:03.683586 1 tasks_processing.go:69] worker 1 listening for tasks. I0507 15:33:03.683593 1 tasks_processing.go:69] worker 0 listening for tasks. I0507 15:33:03.683596 1 tasks_processing.go:69] worker 3 listening for tasks. I0507 15:33:03.683599 1 tasks_processing.go:69] worker 2 listening for tasks. I0507 15:33:03.683607 1 tasks_processing.go:69] worker 6 listening for tasks. I0507 15:33:03.683607 1 tasks_processing.go:69] worker 17 listening for tasks. I0507 15:33:03.683609 1 tasks_processing.go:69] worker 10 listening for tasks. I0507 15:33:03.683613 1 tasks_processing.go:69] worker 11 listening for tasks. I0507 15:33:03.683615 1 tasks_processing.go:69] worker 4 listening for tasks. I0507 15:33:03.683620 1 tasks_processing.go:69] worker 15 listening for tasks. I0507 15:33:03.683623 1 tasks_processing.go:69] worker 14 listening for tasks. I0507 15:33:03.683626 1 tasks_processing.go:69] worker 16 listening for tasks. I0507 15:33:03.683629 1 tasks_processing.go:69] worker 7 listening for tasks. I0507 15:33:03.683622 1 tasks_processing.go:69] worker 5 listening for tasks. I0507 15:33:03.683632 1 tasks_processing.go:69] worker 12 listening for tasks. I0507 15:33:03.683635 1 tasks_processing.go:69] worker 53 listening for tasks. I0507 15:33:03.683638 1 tasks_processing.go:71] worker 12 working on nodenetworkconfigurationpolicies task. I0507 15:33:03.683640 1 tasks_processing.go:69] worker 13 listening for tasks. I0507 15:33:03.683643 1 tasks_processing.go:69] worker 9 listening for tasks. I0507 15:33:03.683643 1 tasks_processing.go:69] worker 39 listening for tasks. I0507 15:33:03.683647 1 tasks_processing.go:69] worker 27 listening for tasks. I0507 15:33:03.683646 1 tasks_processing.go:69] worker 54 listening for tasks. I0507 15:33:03.683653 1 tasks_processing.go:69] worker 42 listening for tasks. I0507 15:33:03.683638 1 tasks_processing.go:71] worker 7 working on ceph_cluster task. I0507 15:33:03.683657 1 tasks_processing.go:69] worker 48 listening for tasks. I0507 15:33:03.683660 1 tasks_processing.go:69] worker 18 listening for tasks. I0507 15:33:03.683662 1 tasks_processing.go:69] worker 56 listening for tasks. I0507 15:33:03.683656 1 tasks_processing.go:69] worker 38 listening for tasks. I0507 15:33:03.683670 1 tasks_processing.go:69] worker 19 listening for tasks. I0507 15:33:03.683655 1 tasks_processing.go:69] worker 55 listening for tasks. I0507 15:33:03.683667 1 tasks_processing.go:69] worker 49 listening for tasks. I0507 15:33:03.683678 1 tasks_processing.go:69] worker 28 listening for tasks. I0507 15:33:03.683673 1 tasks_processing.go:69] worker 50 listening for tasks. I0507 15:33:03.683674 1 tasks_processing.go:69] worker 57 listening for tasks. I0507 15:33:03.683689 1 tasks_processing.go:69] worker 24 listening for tasks. I0507 15:33:03.683690 1 tasks_processing.go:69] worker 59 listening for tasks. I0507 15:33:03.683689 1 tasks_processing.go:69] worker 46 listening for tasks. I0507 15:33:03.683698 1 tasks_processing.go:69] worker 25 listening for tasks. I0507 15:33:03.683692 1 tasks_processing.go:69] worker 23 listening for tasks. I0507 15:33:03.683703 1 tasks_processing.go:69] worker 43 listening for tasks. I0507 15:33:03.683706 1 tasks_processing.go:69] worker 26 listening for tasks. I0507 15:33:03.683707 1 tasks_processing.go:69] worker 61 listening for tasks. I0507 15:33:03.683709 1 tasks_processing.go:69] worker 41 listening for tasks. I0507 15:33:03.683683 1 tasks_processing.go:69] worker 58 listening for tasks. I0507 15:33:03.683716 1 tasks_processing.go:69] worker 33 listening for tasks. I0507 15:33:03.683694 1 tasks_processing.go:69] worker 29 listening for tasks. I0507 15:33:03.683716 1 tasks_processing.go:69] worker 45 listening for tasks. I0507 15:33:03.683722 1 tasks_processing.go:69] worker 36 listening for tasks. I0507 15:33:03.683726 1 tasks_processing.go:69] worker 22 listening for tasks. I0507 15:33:03.683727 1 tasks_processing.go:69] worker 34 listening for tasks. I0507 15:33:03.683730 1 tasks_processing.go:71] worker 1 working on monitoring_persistent_volumes task. I0507 15:33:03.683728 1 tasks_processing.go:69] worker 37 listening for tasks. I0507 15:33:03.683735 1 tasks_processing.go:71] worker 27 working on ingress_certificates task. I0507 15:33:03.683738 1 tasks_processing.go:71] worker 4 working on ingress task. I0507 15:33:03.683738 1 tasks_processing.go:71] worker 13 working on image_registries task. I0507 15:33:03.683945 1 tasks_processing.go:71] worker 48 working on jaegers task. I0507 15:33:03.684003 1 tasks_processing.go:71] worker 18 working on storage_cluster task. I0507 15:33:03.684004 1 tasks_processing.go:71] worker 5 working on machine_healthchecks task. I0507 15:33:03.683916 1 tasks_processing.go:71] worker 9 working on machine_config_pools task. I0507 15:33:03.684037 1 tasks_processing.go:71] worker 25 working on proxies task. I0507 15:33:03.684079 1 tasks_processing.go:71] worker 42 working on tsdb_status task. I0507 15:33:03.684123 1 tasks_processing.go:71] worker 14 working on nodes task. I0507 15:33:03.684202 1 tasks_processing.go:71] worker 29 working on install_plans task. I0507 15:33:03.684211 1 tasks_processing.go:71] worker 26 working on node_logs task. I0507 15:33:03.683727 1 tasks_processing.go:71] worker 17 working on image_pruners task. I0507 15:33:03.684230 1 tasks_processing.go:71] worker 16 working on openshift_machine_api_events task. I0507 15:33:03.684245 1 tasks_processing.go:71] worker 6 working on support_secret task. I0507 15:33:03.683735 1 tasks_processing.go:71] worker 3 working on openstack_version task. I0507 15:33:03.684326 1 tasks_processing.go:71] worker 61 working on openstack_dataplanenodesets task. I0507 15:33:03.684332 1 tasks_processing.go:71] worker 15 working on openshift_logging task. I0507 15:33:03.683650 1 tasks_processing.go:69] worker 47 listening for tasks. I0507 15:33:03.683740 1 tasks_processing.go:71] worker 2 working on pod_network_connectivity_checks task. I0507 15:33:03.684431 1 tasks_processing.go:71] worker 45 working on service_accounts task. I0507 15:33:03.684465 1 tasks_processing.go:71] worker 47 working on qemu_kubevirt_launcher_logs task. I0507 15:33:03.683699 1 tasks_processing.go:69] worker 30 listening for tasks. I0507 15:33:03.684536 1 tasks_processing.go:71] worker 30 working on sap_datahubs task. I0507 15:33:03.684556 1 tasks_processing.go:71] worker 36 working on mutating_webhook_configurations task. I0507 15:33:03.684671 1 tasks_processing.go:71] worker 41 working on silenced_alerts task. I0507 15:33:03.683694 1 tasks_processing.go:69] worker 52 listening for tasks. I0507 15:33:03.684114 1 tasks_processing.go:71] worker 23 working on dvo_metrics task. I0507 15:33:03.683711 1 tasks_processing.go:69] worker 44 listening for tasks. I0507 15:33:03.683711 1 tasks_processing.go:69] worker 20 listening for tasks. I0507 15:33:03.683712 1 tasks_processing.go:69] worker 32 listening for tasks. I0507 15:33:03.683717 1 tasks_processing.go:69] worker 62 listening for tasks. I0507 15:33:03.683718 1 tasks_processing.go:69] worker 21 listening for tasks. I0507 15:33:03.683719 1 tasks_processing.go:69] worker 35 listening for tasks. I0507 15:33:03.683723 1 tasks_processing.go:69] worker 63 listening for tasks. I0507 15:33:03.683686 1 tasks_processing.go:69] worker 51 listening for tasks. I0507 15:33:03.683728 1 tasks_processing.go:71] worker 10 working on aggregated_monitoring_cr_names task. I0507 15:33:03.683733 1 tasks_processing.go:71] worker 39 working on metrics task. I0507 15:33:03.683733 1 tasks_processing.go:71] worker 11 working on infrastructures task. I0507 15:33:03.683739 1 tasks_processing.go:71] worker 0 working on active_alerts task. I0507 15:33:03.683719 1 tasks_processing.go:69] worker 40 listening for tasks. I0507 15:33:03.684019 1 tasks_processing.go:71] worker 54 working on operators task. I0507 15:33:03.684119 1 tasks_processing.go:71] worker 43 working on openstack_dataplanedeployments task. I0507 15:33:03.684124 1 tasks_processing.go:71] worker 33 working on config_maps task. I0507 15:33:03.683707 1 tasks_processing.go:69] worker 31 listening for tasks. I0507 15:33:03.684765 1 tasks_processing.go:71] worker 28 working on networks task. I0507 15:33:03.685363 1 tasks_processing.go:71] worker 52 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0507 15:33:03.685393 1 tasks_processing.go:71] worker 40 working on sap_pods task. I0507 15:33:03.685446 1 tasks_processing.go:71] worker 32 working on cluster_apiserver task. I0507 15:33:03.685611 1 tasks_processing.go:71] worker 44 working on node_features task. I0507 15:33:03.685617 1 tasks_processing.go:71] worker 21 working on schedulers task. I0507 15:33:03.683699 1 tasks_processing.go:69] worker 60 listening for tasks. I0507 15:33:03.684823 1 tasks_processing.go:71] worker 56 working on storage_classes task. I0507 15:33:03.685417 1 tasks_processing.go:71] worker 53 working on crds task. I0507 15:33:03.684830 1 tasks_processing.go:71] worker 38 working on container_runtime_configs task. I0507 15:33:03.684864 1 tasks_processing.go:71] worker 24 working on overlapping_namespace_uids task. I0507 15:33:03.684994 1 tasks_processing.go:71] worker 50 working on machine_autoscalers task. I0507 15:33:03.684999 1 tasks_processing.go:71] worker 57 working on machine_sets task. I0507 15:33:03.685010 1 tasks_processing.go:71] worker 19 working on feature_gates task. I0507 15:33:03.685020 1 tasks_processing.go:71] worker 55 working on cost_management_metrics_configs task. I0507 15:33:03.685048 1 tasks_processing.go:71] worker 49 working on machine_configs task. I0507 15:33:03.683638 1 tasks_processing.go:69] worker 8 listening for tasks. I0507 15:33:03.685072 1 tasks_processing.go:71] worker 59 working on operators_pods_and_events task. I0507 15:33:03.685082 1 tasks_processing.go:71] worker 58 working on clusterroles task. I0507 15:33:03.685121 1 tasks_processing.go:71] worker 46 working on image task. I0507 15:33:03.685168 1 tasks_processing.go:71] worker 22 working on version task. I0507 15:33:03.685434 1 tasks_processing.go:71] worker 37 working on nodenetworkstates task. I0507 15:33:03.685459 1 tasks_processing.go:71] worker 63 working on sap_config task. I0507 15:33:03.686595 1 tasks_processing.go:71] worker 20 working on pdbs task. I0507 15:33:03.686857 1 tasks_processing.go:71] worker 34 working on machines task. I0507 15:33:03.686950 1 tasks_processing.go:71] worker 60 working on openstack_controlplanes task. I0507 15:33:03.687035 1 tasks_processing.go:71] worker 31 working on olm_operators task. I0507 15:33:03.687122 1 tasks_processing.go:71] worker 62 working on authentication task. I0507 15:33:03.687134 1 tasks_processing.go:71] worker 35 working on validating_webhook_configurations task. I0507 15:33:03.687147 1 tasks_processing.go:71] worker 51 working on certificate_signing_requests task. I0507 15:33:03.687298 1 tasks_processing.go:71] worker 7 working on lokistack task. I0507 15:33:03.687310 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 3.636713ms to process 0 records I0507 15:33:03.687479 1 tasks_processing.go:71] worker 12 working on oauths task. I0507 15:33:03.687515 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 3.831984ms to process 0 records I0507 15:33:03.688106 1 tasks_processing.go:71] worker 8 working on container_images task. I0507 15:33:03.690010 1 tasks_processing.go:74] worker 5 stopped. E0507 15:33:03.690028 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0507 15:33:03.690040 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 5.978427ms to process 0 records E0507 15:33:03.691440 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.148.97:9091: connect: connection refused E0507 15:33:03.691438 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.148.97:9091: connect: connection refused I0507 15:33:03.691462 1 tasks_processing.go:74] worker 42 stopped. E0507 15:33:03.691471 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.148.97:9091: connect: connection refused I0507 15:33:03.691483 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 7.322912ms to process 0 records E0507 15:33:03.691497 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.148.97:9091: connect: connection refused I0507 15:33:03.691519 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 6.227807ms to process 0 records I0507 15:33:03.691529 1 tasks_processing.go:74] worker 39 stopped. I0507 15:33:03.692242 1 tasks_processing.go:74] worker 2 stopped. E0507 15:33:03.692254 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0507 15:33:03.692261 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 7.806327ms to process 0 records I0507 15:33:03.701481 1 tasks_processing.go:74] worker 3 stopped. I0507 15:33:03.701494 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 17.211381ms to process 0 records I0507 15:33:03.701501 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 17.519621ms to process 0 records I0507 15:33:03.701505 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 17.456052ms to process 0 records I0507 15:33:03.701509 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 17.461945ms to process 0 records I0507 15:33:03.701514 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 17.135047ms to process 0 records I0507 15:33:03.701518 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 17.151543ms to process 0 records I0507 15:33:03.701517 1 tasks_processing.go:74] worker 48 stopped. I0507 15:33:03.701522 1 tasks_processing.go:74] worker 15 stopped. I0507 15:33:03.701523 1 tasks_processing.go:74] worker 9 stopped. I0507 15:33:03.701526 1 tasks_processing.go:74] worker 61 stopped. I0507 15:33:03.701527 1 tasks_processing.go:74] worker 18 stopped. I0507 15:33:03.701658 1 tasks_processing.go:74] worker 25 stopped. I0507 15:33:03.701712 1 controller.go:129] Initializing last reported time to 0001-01-01T00:00:00Z I0507 15:33:03.701727 1 controller.go:368] The initial operator extension status is healthy I0507 15:33:03.701750 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0507 15:33:03.701757 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0507 15:33:03.701761 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0507 15:33:03.701778 1 controller.go:531] The operator is still being initialized I0507 15:33:03.701785 1 controller.go:554] The operator is healthy I0507 15:33:03.701799 1 recorder.go:75] Recording config/proxy with fingerprint=f4c0ecb951368b9b23c0d1b701779930472eec6bbd2e1b5195e476f1aff2ab86 I0507 15:33:03.701819 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 17.592821ms to process 1 records I0507 15:33:03.701839 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0507 15:33:03.701841 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0507 15:33:03.702063 1 operator.go:328] started I0507 15:33:03.702084 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0507 15:33:03.707356 1 tasks_processing.go:74] worker 30 stopped. I0507 15:33:03.707370 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 22.809427ms to process 0 records E0507 15:33:03.707375 1 gather_node_features.go:86] GatherNodeFeatures: NodeFeatures resource not found in openshift-nfd namespace (may not be installed) I0507 15:33:03.707384 1 tasks_processing.go:74] worker 44 stopped. I0507 15:33:03.707391 1 gather.go:177] gatherer "clusterconfig" function "node_features" took 21.762363ms to process 0 records I0507 15:33:03.707691 1 tasks_processing.go:74] worker 32 stopped. I0507 15:33:03.707875 1 recorder.go:75] Recording config/apiserver with fingerprint=ddecfb1c15970a1d1ee7a09ae6558aa23a49e8db4cb058ba386d766bd8551078 I0507 15:33:03.707894 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 22.183955ms to process 1 records I0507 15:33:03.707977 1 tasks_processing.go:74] worker 28 stopped. I0507 15:33:03.708048 1 recorder.go:75] Recording config/network with fingerprint=9bde99fe4d466ee4b4dd1f84e9047a896144aa9309d16014567cf82814921429 I0507 15:33:03.708057 1 gather.go:177] gatherer "clusterconfig" function "networks" took 22.42105ms to process 1 records I0507 15:33:03.711315 1 tasks_processing.go:74] worker 34 stopped. E0507 15:33:03.711331 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0507 15:33:03.711360 1 gather.go:177] gatherer "clusterconfig" function "machines" took 24.436995ms to process 0 records I0507 15:33:03.711479 1 tasks_processing.go:74] worker 4 stopped. I0507 15:33:03.711633 1 recorder.go:75] Recording config/ingress with fingerprint=e0b83900347754faac5c5c8b4dd27d0263a5e51064720d8d82cde4d8d3c6a7e3 I0507 15:33:03.711643 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 27.729533ms to process 1 records I0507 15:33:03.712101 1 tasks_processing.go:74] worker 11 stopped. I0507 15:33:03.712702 1 recorder.go:75] Recording config/infrastructure with fingerprint=d48a8c0fa2796f52b6eee0c1f5fb079c03c40d4a45c03c3384b133d4db402dc8 I0507 15:33:03.712714 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 26.857429ms to process 1 records I0507 15:33:03.713357 1 tasks_processing.go:74] worker 16 stopped. I0507 15:33:03.713369 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 29.117811ms to process 0 records I0507 15:33:03.713604 1 tasks_processing.go:74] worker 21 stopped. I0507 15:33:03.713675 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=de70359c3621f910864b3c800c1ce6f7d840e9d4fe98d123bea31d72a2632697 I0507 15:33:03.713688 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 27.972509ms to process 1 records I0507 15:33:03.714414 1 tasks_processing.go:74] worker 60 stopped. I0507 15:33:03.714434 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 27.448239ms to process 0 records I0507 15:33:03.714488 1 tasks_processing.go:74] worker 6 stopped. E0507 15:33:03.714510 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0507 15:33:03.714522 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 30.219679ms to process 0 records I0507 15:33:03.714695 1 tasks_processing.go:74] worker 43 stopped. I0507 15:33:03.714706 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 29.204356ms to process 0 records I0507 15:33:03.714936 1 tasks_processing.go:74] worker 38 stopped. I0507 15:33:03.714953 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 29.190287ms to process 0 records I0507 15:33:03.714964 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 30.709674ms to process 0 records I0507 15:33:03.714974 1 tasks_processing.go:74] worker 26 stopped. I0507 15:33:03.715094 1 tasks_processing.go:74] worker 7 stopped. I0507 15:33:03.715110 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 27.782342ms to process 0 records I0507 15:33:03.715362 1 gather_logs.go:145] no pods in namespace were found I0507 15:33:03.715423 1 tasks_processing.go:74] worker 47 stopped. I0507 15:33:03.715442 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 30.944215ms to process 0 records I0507 15:33:03.716363 1 tasks_processing.go:74] worker 62 stopped. I0507 15:33:03.717201 1 recorder.go:75] Recording config/authentication with fingerprint=cdd352ca7a0d26efa7872c8d9c5ef8a3697c4c7b285bb4110cb16889a76f504d I0507 15:33:03.717226 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 29.223202ms to process 1 records I0507 15:33:03.718707 1 tasks_processing.go:74] worker 56 stopped. I0507 15:33:03.718827 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=3a0d1144092804ab393c4e27ed637bf3e2f6bf35b2cbc8105c0632a97a05fc2f I0507 15:33:03.718856 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=390754dbdf1275a0a59d0c41cead147ece448091a6d89a1f63ab50c9a665cdbd I0507 15:33:03.718870 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 33.055982ms to process 2 records I0507 15:33:03.718881 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 34.995223ms to process 0 records I0507 15:33:03.718968 1 tasks_processing.go:74] worker 1 stopped. I0507 15:33:03.718979 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=46f4ee03ee5aaf4c67959078933329d85bcab634acdebe3be85843ea952f6024 I0507 15:33:03.719001 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=7c068c2a5573917e88d9166a5275e7f66aff16b608a562b462314e26212d6ddf I0507 15:33:03.719022 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=01bb4ad50516d88af1a2ebf227b02793a0376c48e18f571ae6f75d5a07283ae5 I0507 15:33:03.719036 1 tasks_processing.go:74] worker 20 stopped. I0507 15:33:03.719100 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=f924884c9e54ef555c0118725492982104748a73ecab3c19f918f58bff597b9b I0507 15:33:03.719135 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=7a80c39362c2d76dfa967b09ce6041fa741cbc17daae42ff780a88e38f75393d I0507 15:33:03.719163 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=56ec2e56ce50010feed01ce6197f16a9cac3465c8db11487518bcbaf04878a9b I0507 15:33:03.719193 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=5dc0a51124a12e60b871bae7c3605744f756fedfad433ae9080a239b97997942 I0507 15:33:03.719216 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=6a14e9f21218286d511b4af0a30b1bf796e27ffdaef9fa0e83b035dcd8235933 I0507 15:33:03.719234 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=39e084176f78397c9ac12d23bdb5fffc724ef6e903aef9b06d403e1ab30e7dcc I0507 15:33:03.719250 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=21592d14df53db846f73015d4d8cdb7885c1348021dae472901b9cb99156051c I0507 15:33:03.719263 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=09c075c8942af8fe51e79071a8e70874899bb0f8677267178818ab6184c15fff I0507 15:33:03.719269 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 32.178268ms to process 11 records I0507 15:33:03.719280 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 32.923439ms to process 0 records I0507 15:33:03.719285 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 33.360644ms to process 0 records I0507 15:33:03.719293 1 tasks_processing.go:74] worker 57 stopped. I0507 15:33:03.719301 1 tasks_processing.go:74] worker 40 stopped. I0507 15:33:03.719354 1 tasks_processing.go:74] worker 12 stopped. I0507 15:33:03.719506 1 recorder.go:75] Recording config/oauth with fingerprint=51e5b8122a36557951c8e641768d2be9320e80814ed49cbb880eb0501f007187 I0507 15:33:03.719518 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 31.431155ms to process 1 records I0507 15:33:03.719524 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 32.906959ms to process 0 records I0507 15:33:03.719531 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 32.030439ms to process 0 records I0507 15:33:03.719538 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 33.364162ms to process 0 records I0507 15:33:03.719544 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 33.558449ms to process 0 records I0507 15:33:03.719551 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 33.424676ms to process 0 records I0507 15:33:03.719561 1 tasks_processing.go:74] worker 37 stopped. I0507 15:33:03.719561 1 tasks_processing.go:74] worker 31 stopped. I0507 15:33:03.719566 1 tasks_processing.go:74] worker 63 stopped. I0507 15:33:03.719573 1 tasks_processing.go:74] worker 50 stopped. I0507 15:33:03.719575 1 tasks_processing.go:74] worker 55 stopped. I0507 15:33:03.719695 1 tasks_processing.go:74] worker 17 stopped. I0507 15:33:03.720008 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=dda7856dba53a3232632da8be3969a910648d3b728d61e67348ae2d03abb2419 I0507 15:33:03.720028 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 35.458087ms to process 1 records I0507 15:33:03.720545 1 tasks_processing.go:74] worker 14 stopped. I0507 15:33:03.720920 1 recorder.go:75] Recording config/node/ip-10-0-0-54.ec2.internal with fingerprint=0277e2214ae119e4460ac1b912eab22688aff58f42e94d828899a00757025d19 I0507 15:33:03.720988 1 recorder.go:75] Recording config/node/ip-10-0-1-119.ec2.internal with fingerprint=9d170bd8a6093009ce24cb4b2d036b38dac456fa767522dc7f4caac6a27d846f I0507 15:33:03.721051 1 recorder.go:75] Recording config/node/ip-10-0-2-215.ec2.internal with fingerprint=60027964923d4e2e730b02e13b0b77facea9efbb75b892c04c5242da2538e9e0 I0507 15:33:03.721061 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 36.388877ms to process 3 records I0507 15:33:03.721155 1 tasks_processing.go:74] worker 13 stopped. I0507 15:33:03.721407 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=709c80c7b1fe90873db53e84b33ab91ccadc100112cdf0864a4c10bd6f1d35ee I0507 15:33:03.721418 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 37.038794ms to process 1 records W0507 15:33:03.721488 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0507 15:33:03.722840 1 tasks_processing.go:74] worker 35 stopped. I0507 15:33:03.722961 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=f188b6e7f725fa1a0c218355f7a93165d73ae319486f5f3f81cfcc87d5c671b9 I0507 15:33:03.723014 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=8c003a4f9b2d2665a4b43b745e83bf1c61814efa4c50dcb3523300f39b2edb35 I0507 15:33:03.723096 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=eca68d7f5a9a224ae22f9b5523548c5cf06d3916d7bbdc656478eb95f0ea2d37 I0507 15:33:03.723129 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=84df00efa982e33a5f970eaa4f209d2ff14a32ea21d05140319d29680a736426 I0507 15:33:03.723151 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=21cb9b47e65b5a6a2b19daa5508adb181153bcfc4a1c7e09cab88aea53398b34 I0507 15:33:03.723181 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=a783b4b351ed99dbab8f1b3ccafa45f82bd3884d6b6d45a643b5da5314cd2ed0 I0507 15:33:03.723209 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=524af989b29ce449f386ec43ec4cbdce70bc7202f977f855e251c261cf538e13 I0507 15:33:03.723410 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=69ecdb76058b26e717d87b9c66a181b14acadd532ec282973250d016fd526f00 I0507 15:33:03.723466 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=fe610acbabc2d31683345b822e2fac015ca2fc89036fb9bf5206e61bafdc8112 I0507 15:33:03.723719 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=b4be9cb42112807e97f6256596433cbb25adea02901bcef2a9366b13604c716d I0507 15:33:03.723907 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=1b2e919ad6c3374b9129d2ff7b8f7c76fef1e9d180cadd46f81b49fda03aa87c I0507 15:33:03.724003 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=c9f2857e49e4a2d4834c16671317174edd06ff5160bac685c3109b273e1e13d0 I0507 15:33:03.724086 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=bcf08d86fbfd8e56f1f206fe50a42d4cb20399788b708ccb37ebee15c8973827 I0507 15:33:03.724143 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 35.671061ms to process 13 records I0507 15:33:03.727921 1 tasks_processing.go:74] worker 36 stopped. I0507 15:33:03.728034 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=95f84c009d470812f3032c3fc8e3c7258d4e6106b957373451884e30247b3b06 I0507 15:33:03.728084 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=abe9a85d8a8d7a50f5068103378a211cbc31823b838d50df2bffa7773e6a4e55 I0507 15:33:03.728121 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=1115a413f86964ddf473dec7e9367041b2caeac3d406348e376f0a9903c970a0 I0507 15:33:03.728130 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 43.342664ms to process 3 records I0507 15:33:03.728162 1 tasks_processing.go:74] worker 49 stopped. E0507 15:33:03.728169 1 gather.go:140] gatherer "clusterconfig" function "machine_configs" failed with the error: getting MachineConfigPools failed: the server could not find the requested resource (get machineconfigpools.machineconfiguration.openshift.io) I0507 15:33:03.728183 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0507 15:33:03.728189 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 42.174802ms to process 1 records I0507 15:33:03.728305 1 recorder.go:75] Recording config/featuregate with fingerprint=823a88d6aa97a97d8029898b391c802421c3332b9af934f1fac69e29454a747b I0507 15:33:03.728303 1 tasks_processing.go:74] worker 19 stopped. I0507 15:33:03.728320 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 42.258614ms to process 1 records I0507 15:33:03.728818 1 controller.go:254] Source periodic-workloads *controllerstatus.Simple is not ready I0507 15:33:03.728842 1 controller.go:254] Source scaController *sca.Controller is not ready I0507 15:33:03.728847 1 controller.go:254] Source clusterTransferController *clustertransfer.Controller is not ready I0507 15:33:03.728854 1 controller.go:254] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0507 15:33:03.728858 1 controller.go:254] Source periodic-conditional *controllerstatus.Simple is not ready I0507 15:33:03.728885 1 controller.go:531] The operator is still being initialized I0507 15:33:03.728899 1 controller.go:554] The operator is healthy I0507 15:33:03.729019 1 tasks_processing.go:74] worker 46 stopped. I0507 15:33:03.729121 1 recorder.go:75] Recording config/image with fingerprint=ea2faf56e5686bde5ac27d736704944a32fb27c2ece91525a708abab8476c32a I0507 15:33:03.729139 1 gather.go:177] gatherer "clusterconfig" function "image" took 42.975152ms to process 1 records I0507 15:33:03.730007 1 tasks_processing.go:74] worker 58 stopped. I0507 15:33:03.730180 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=03304d0f1b7c40b043363363cafaa75383317caafe798e5a787fa173bf200955 I0507 15:33:03.730270 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=e53865cf43adc0c10d9df340bc1db959f025fc8c43cb4167d8db497b31bd75e5 I0507 15:33:03.730279 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 43.976566ms to process 2 records I0507 15:33:03.730291 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0507 15:33:03.730297 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 44.428367ms to process 1 records I0507 15:33:03.730303 1 tasks_processing.go:74] worker 24 stopped. I0507 15:33:03.731124 1 tasks_processing.go:74] worker 51 stopped. I0507 15:33:03.731134 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 43.920954ms to process 0 records I0507 15:33:03.737454 1 tasks_processing.go:74] worker 53 stopped. I0507 15:33:03.739077 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=c2dfc8de79ee43025abee4e48b9cce4555ff1a9a4fee65b19356de7ecac8f68b I0507 15:33:03.739279 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=9592206a7f14e5021496dc3c072386404bdade260348d9e1e60a196f47b7adcc I0507 15:33:03.739288 1 gather.go:177] gatherer "clusterconfig" function "crds" took 51.728466ms to process 2 records I0507 15:33:03.739644 1 tasks_processing.go:74] worker 10 stopped. I0507 15:33:03.739660 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 54.395867ms to process 0 records I0507 15:33:03.741666 1 tasks_processing.go:74] worker 59 stopped. I0507 15:33:03.741690 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 55.645608ms to process 0 records I0507 15:33:03.742649 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0507 15:33:03.742654 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0507 15:33:03.742736 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0507 15:33:03.747997 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0507 15:33:03.752053 1 base_controller.go:82] Caches are synced for ConfigController I0507 15:33:03.752128 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0507 15:33:03.760788 1 tasks_processing.go:74] worker 52 stopped. I0507 15:33:03.760805 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 75.371703ms to process 0 records I0507 15:33:03.761631 1 configmapobserver.go:84] configmaps "insights-config" not found I0507 15:33:03.773572 1 tasks_processing.go:74] worker 33 stopped. E0507 15:33:03.773587 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0507 15:33:03.773593 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0507 15:33:03.773597 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0507 15:33:03.773608 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=ad286723f58bdcfc37aeba3ec5b4110c08e1af59cd34d14b4bfaab02d18e4856 I0507 15:33:03.773645 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=d476c7d3f5b104863f08f481b1264dcc68cc272ecefb0ecb709b18a6afab034d I0507 15:33:03.773653 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0507 15:33:03.773657 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=2ab5ab7b1b10d7fcf1197bb24dea7c90f400e4effc18c7356873209d54fdf84b I0507 15:33:03.773668 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0507 15:33:03.773705 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0507 15:33:03.773713 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0507 15:33:03.773731 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 88.056032ms to process 7 records I0507 15:33:03.775805 1 tasks_processing.go:74] worker 27 stopped. E0507 15:33:03.775822 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0507 15:33:03.775828 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2q55t83gi7r3iblf5ke81p881m72e003-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2q55t83gi7r3iblf5ke81p881m72e003-primary-cert-bundle-secret" not found I0507 15:33:03.775882 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=db13ba8b40125ce0bcadd673e4a98890ec3a1ecc3f26a2db2ddc95504ff1ddce I0507 15:33:03.775895 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 92.059593ms to process 1 records I0507 15:33:03.778499 1 tasks_processing.go:74] worker 8 stopped. I0507 15:33:03.779824 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-66dbbb6468-b2hsr with fingerprint=9f684fdf398cc96506f9d9da28ab74664e8b70989291862f7b72ef186c472108 I0507 15:33:03.779958 1 recorder.go:75] Recording config/pod/openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-2-215.ec2.internal with fingerprint=31953f0a4e9457462af7f9007e89e67c937a31bf75f57367d494f2d842db74c3 I0507 15:33:03.780030 1 recorder.go:75] Recording config/running_containers with fingerprint=f2b1bce0e687652d080803fb1502403570bf955bd8d3db4e783b4a9b73c40657 I0507 15:33:03.780040 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 90.371903ms to process 3 records I0507 15:33:03.797500 1 tasks_processing.go:74] worker 22 stopped. I0507 15:33:03.797765 1 recorder.go:75] Recording config/version with fingerprint=222034f86fe840d20d4087a89c17ca2aef6b757e468549f8c8a14f5750077231 I0507 15:33:03.797777 1 recorder.go:75] Recording config/id with fingerprint=3f7f3410ce48a30a799fbf9df08364cf10d9c416bb8b9e15343094fdd554e716 I0507 15:33:03.797784 1 gather.go:177] gatherer "clusterconfig" function "version" took 111.431856ms to process 2 records I0507 15:33:03.802848 1 base_controller.go:82] Caches are synced for LoggingSyncer I0507 15:33:03.802864 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0507 15:33:03.806478 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0507 15:33:03.811573 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0507 15:33:03.814549 1 cluster_transfer.go:102] no available accepted cluster transfer I0507 15:33:03.814565 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0507 15:33:04.104182 1 sca.go:266] etc-pki-entitlement secret successfully created I0507 15:33:04.104201 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated W0507 15:33:04.721749 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0507 15:33:04.740761 1 tasks_processing.go:74] worker 41 stopped. I0507 15:33:04.740782 1 recorder.go:75] Recording config/silenced_alerts.json with fingerprint=37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570 I0507 15:33:04.740799 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 1.055695797s to process 1 records I0507 15:33:04.740822 1 tasks_processing.go:74] worker 0 stopped. I0507 15:33:04.740857 1 recorder.go:75] Recording config/alerts with fingerprint=785e8951e8489d873a620077756465cf560d9167e0f66eda7337e5e293c54450 I0507 15:33:04.740870 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 1.055580458s to process 1 records W0507 15:33:05.722001 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0507 15:33:06.154794 1 tasks_processing.go:74] worker 54 stopped. I0507 15:33:06.154913 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=cba1d7fe6a79c4e0450960189832e300f15063307ed89461dc66ddfb241ccbe4 I0507 15:33:06.154963 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0507 15:33:06.154997 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=69e159744c202713f3bc8388c73657e471bf5ecdc29a25026d2b162689d165d3 I0507 15:33:06.155007 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0507 15:33:06.155029 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=3e95092b0104420e00d2baeef80821de2dbca2ed32e82eeb09ee381def5e29a1 I0507 15:33:06.155044 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0507 15:33:06.155064 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=8038888a2438883497cec4c6afa2d8e48ed6d1b1ea29c24a3e777b51f279a87b I0507 15:33:06.155092 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=4bcc07371fdaae7b9c86437243e58e84d3e3fb5b7a0f69dd317d227f0196fe7e I0507 15:33:06.155114 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=b3bb3f39a771d35377e3425e357524fe990a5961cf216a58b707502cc23e3c9b I0507 15:33:06.155144 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=080433e1315f545a39c05978af23debf88d64c1052b438c607c8f2c1a5c32829 I0507 15:33:06.155426 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=4248c0176be2a860c80ad2fc42693e3fca42333e8af2d65e7e5a36f46bd6e779 I0507 15:33:06.155448 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=a44d1db13d0266918aa57ab2b0a563e07f193360bba9f147496f6620946957b1 I0507 15:33:06.155456 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0507 15:33:06.155473 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=08bac2b31b0cde115e34937d0874c035a051e61d2f8f32e964a0dbd1e1fb9ca5 I0507 15:33:06.155484 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0507 15:33:06.155500 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=ea3c62a6495623f5d1fa3b385f93fe4fe88c28cebf9db6fbf34cf018999a6be3 I0507 15:33:06.155509 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0507 15:33:06.155527 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=ccdfd9607f2b69eb856b23ad59637a82fc9ab1a6d9e104e1d907f78fc0720cf6 I0507 15:33:06.155537 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0507 15:33:06.155558 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=330661c88d5023206726d7bb1831530a608e2e22930410052029ddf3af981e69 I0507 15:33:06.155659 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=9184f9d1715cb88899857437fb01a920ca29f54f07a2c8ab4297e9de7f733729 I0507 15:33:06.155669 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0507 15:33:06.155676 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0507 15:33:06.155711 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0507 15:33:06.155735 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=307ef54e50fc14ebf41c19709136329b66a656fe88e82af9896acc664cd046c1 I0507 15:33:06.155757 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=11af788ebb2ce71fad2a8f2978c1657b15e1454183c5df2d9c50ecdbea24c7d0 I0507 15:33:06.155767 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0507 15:33:06.155783 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=1daefaab451886f5369332778b44b422f9c545cfa3931572aa453e722b243a19 I0507 15:33:06.155794 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0507 15:33:06.155811 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=f06dd2eca1a85031a0c9efc7a57f2785f974287b70d70cfca284bead8ba948a7 I0507 15:33:06.155820 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0507 15:33:06.155833 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=573cda3004c0c4101ff9bce54b55a501d8140569323f3e8ab37217ac13ffc924 I0507 15:33:06.155845 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=bee6032215603f2779fa978b359c4f5a59f5e33eb5876911dd05aa5c179e6249 I0507 15:33:06.155860 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=04e25ecb96ee8d49d5be79558732acb9e1f766bbbdf40e9281d38eaea15a9929 I0507 15:33:06.155879 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=130dceeed476c176c455b0839730ee16586d5f7fcb5c3f4f63eb31e1386a8970 I0507 15:33:06.155889 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0507 15:33:06.155916 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=e599e15d8b988bd5041480e6004f24d3ea3e8edfbebd951b5005ef5e82600818 I0507 15:33:06.155934 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0507 15:33:06.155941 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0507 15:33:06.155947 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.469126594s to process 39 records W0507 15:33:06.721281 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0507 15:33:07.721332 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0507 15:33:08.722069 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0507 15:33:08.722095 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0507 15:33:08.722112 1 tasks_processing.go:74] worker 23 stopped. E0507 15:33:08.722123 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0507 15:33:08.722134 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0507 15:33:08.722144 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.036972829s to process 1 records I0507 15:33:16.536209 1 tasks_processing.go:74] worker 29 stopped. I0507 15:33:16.536260 1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b I0507 15:33:16.536274 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 12.85198248s to process 1 records I0507 15:33:17.295544 1 tasks_processing.go:74] worker 45 stopped. I0507 15:33:17.295830 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=80e5b054934e37ddf232a6b52185c88ef3149965af63f70b0125fd661e8f707e I0507 15:33:17.295847 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.611086306s to process 1 records E0507 15:33:17.295918 1 periodic.go:254] "Unhandled Error" err="clusterconfig failed after 13.612s with: function \"machine_healthchecks\" failed with an error, function \"tsdb_status\" failed with an error, function \"metrics\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machines\" failed with an error, function \"support_secret\" failed with an error, function \"machine_configs\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0507 15:33:17.297029 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "machine_healthchecks" failed with an error, function "tsdb_status" failed with an error, function "metrics" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machines" failed with an error, function "support_secret" failed with an error, function "machine_configs" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0507 15:33:17.297045 1 periodic.go:216] Running workloads gatherer I0507 15:33:17.297060 1 tasks_processing.go:45] number of workers: 2 I0507 15:33:17.297071 1 tasks_processing.go:69] worker 1 listening for tasks. I0507 15:33:17.297076 1 tasks_processing.go:71] worker 1 working on workload_info task. I0507 15:33:17.297087 1 tasks_processing.go:69] worker 0 listening for tasks. I0507 15:33:17.297104 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0507 15:33:17.303460 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-119.ec2.internal... I0507 15:33:17.303465 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-215.ec2.internal... I0507 15:33:17.303479 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-54.ec2.internal... I0507 15:33:17.330484 1 tasks_processing.go:74] worker 0 stopped. I0507 15:33:17.330501 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 33.369235ms to process 0 records I0507 15:33:22.136359 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 5s I0507 15:33:22.162204 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 24s for image data I0507 15:33:22.172137 1 gather_workloads_info.go:387] No image sha256:730d1b6988025bef0daa3a9a5d8467ec4a26b0382cc52f91c3375b4590d3518a (11ms) I0507 15:33:22.182438 1 gather_workloads_info.go:387] No image sha256:3a1c7ec9bb0db343348170a70697ea3209ed81e22f1259ff694190e5d04ef250 (10ms) I0507 15:33:22.192696 1 gather_workloads_info.go:387] No image sha256:a043239802b3eb8b323d285193d2527fad0ecec98ca91d188a3472a2fac8ae04 (10ms) I0507 15:33:22.203359 1 gather_workloads_info.go:387] No image sha256:8f881f6f0498eaba860d711dc8e9dc066db8728f22abedb82e1dc9a8e1c30046 (11ms) I0507 15:33:22.213248 1 gather_workloads_info.go:387] No image sha256:17aefdc2f4752c23dc8f0947bb01fc34635822eeb18f2d8a5eddf8211b84d450 (10ms) I0507 15:33:22.223662 1 gather_workloads_info.go:387] No image sha256:08c5a78c8a5af04c549e2273aaf4bb452a75bf038d68aa9d01bb2aff66c30e90 (10ms) I0507 15:33:22.233802 1 gather_workloads_info.go:387] No image sha256:25c148fd380b1a9db3f6039d2e0eabc489a954921452391390ba9192b2325678 (10ms) I0507 15:33:22.243623 1 gather_workloads_info.go:387] No image sha256:000105ef5150e7079b90a613fb9e6193e2a6ef9b1908d2dce44f2395d4fd070f (10ms) I0507 15:33:22.253570 1 gather_workloads_info.go:387] No image sha256:5c6d21c3f97366bc7ab57031cc027b67405a684bd804ce364ed5998b0685eaca (10ms) I0507 15:33:22.263361 1 gather_workloads_info.go:387] No image sha256:5808401268394502d335281ea1a294b07210461b986b58f91d7d1f29c0029c6d (10ms) I0507 15:33:22.273771 1 gather_workloads_info.go:387] No image sha256:5aaea0419169e55832cc27acfe0fe3b9513a343d6bdf71d3da1575ed322245d0 (10ms) I0507 15:33:22.372529 1 gather_workloads_info.go:387] No image sha256:15928a5b803fa00781b408dbc4c3bfa7c5287b18584ba31f817def8a4430c725 (99ms) I0507 15:33:22.473257 1 gather_workloads_info.go:387] No image sha256:15677f0b70e6aa2dfaf088e45fc1a425c22bf6fda326b8116f87e88b6694dfab (101ms) I0507 15:33:22.573240 1 gather_workloads_info.go:387] No image sha256:d64bea34bf3e1bb0b3a701c3ff14e66665afc1b050f28124ad7e6888eaec3a81 (100ms) I0507 15:33:22.673979 1 gather_workloads_info.go:387] No image sha256:5940cc34b322325618929b1e0eb3864dae73d54c8f5b474e218dfb28123a61c7 (101ms) I0507 15:33:22.773422 1 gather_workloads_info.go:387] No image sha256:55b1db6038c5beaed54c626e3343b7a8589cc0be8dc41d1a66b4deab766ff520 (99ms) I0507 15:33:22.874170 1 gather_workloads_info.go:387] No image sha256:48f5559439bea030a6423652da448c25fff0c62ba293eb521843c1a8eb89ddb0 (101ms) I0507 15:33:22.972521 1 gather_workloads_info.go:387] No image sha256:1eb6a6ee3c66ac15be6bc21bb6015c595c11c15b9dfd21bd9f5ce7f7b990ebaa (98ms) I0507 15:33:23.073471 1 gather_workloads_info.go:398] Found image sha256:673ebc8cc22c56c8f410e011b2fa950d28cf7b6420e17fdb6580d6cb10523384 (101ms) I0507 15:33:23.175706 1 gather_workloads_info.go:387] No image sha256:875c77e5d144f03fb91d8cee0259f6966683ca88d1bf818dbf4652c16b70312c (102ms) I0507 15:33:23.271865 1 gather_workloads_info.go:387] No image sha256:52db05084f2a2e7d7bf4040e4865df88ec4074efbbe59a78d861be2c5ab2aa40 (96ms) I0507 15:33:23.372266 1 gather_workloads_info.go:387] No image sha256:9d6d63dce784df5863e6a0ba4c11317a26c9d756d4e62d47759c97190a71cce5 (100ms) I0507 15:33:23.474870 1 gather_workloads_info.go:387] No image sha256:5b6212b8f539f08e78417d8a4b7485ca0b4e7927cacd7b752742a28841bc8ccd (103ms) I0507 15:33:23.572500 1 gather_workloads_info.go:387] No image sha256:6c6d412ccf0c3d6a14645a79df6c4658c0f8725d7954cb83c3d2f0116499345a (98ms) I0507 15:33:23.672720 1 gather_workloads_info.go:387] No image sha256:0a4dfb8d4c1b3849319d45b4c54dff26a7238a2c08fcaa121f93073e95ab12e8 (100ms) I0507 15:33:23.772118 1 gather_workloads_info.go:387] No image sha256:0b2e6536a0ca3afc5e64a218087a6287c74b77a359075a385f1949ca5bd51b22 (99ms) I0507 15:33:23.872597 1 gather_workloads_info.go:387] No image sha256:6814f5fced0ef219d06374011c68a11a3da788a764a00a69ade435466d9ee240 (100ms) I0507 15:33:23.973014 1 gather_workloads_info.go:387] No image sha256:48e883a3932aea9457f55cd4628d21397d429d4828f42f95d1c903d9d5395bde (100ms) I0507 15:33:24.072539 1 gather_workloads_info.go:387] No image sha256:53ff8efdd8f563421ebc1859ab021a3bbc1306f82d4ff5d9c21bc60ec5f57898 (99ms) I0507 15:33:24.172502 1 gather_workloads_info.go:387] No image sha256:5ac9c549d65fc1d8bc900773bebee43e9192bcec1bb5fa46afb4597230c16ac7 (100ms) I0507 15:33:24.274369 1 gather_workloads_info.go:398] Found image sha256:23204877f166455ce8f9de9b91a467eeda576f172d5f96124a68aa986c4775d6 (102ms) I0507 15:33:24.373381 1 gather_workloads_info.go:387] No image sha256:ae7d3453fd734ecc865e5f9bb16f29244ebffe6291b77e1d4e496f71eb053174 (99ms) I0507 15:33:24.473773 1 gather_workloads_info.go:387] No image sha256:30e597ec5d6bb96ff70a4f8688c748b659cd4fd5d73d222e8701821d236795c5 (100ms) I0507 15:33:24.573216 1 gather_workloads_info.go:387] No image sha256:5ff204630794311b0b37fa7b197b933ad85d76a481bf7cdb3bcbada08f0cdcbf (99ms) I0507 15:33:24.672695 1 gather_workloads_info.go:387] No image sha256:9ef6dc311b880a6442b7a3a19006ddd955c7da36850bf000f5c9b68fd0862dd4 (99ms) I0507 15:33:24.772223 1 gather_workloads_info.go:387] No image sha256:3380fecebaeee139c1607fbcd9346cebc92806679568e2244a8905ef0f52aad2 (100ms) I0507 15:33:24.872729 1 gather_workloads_info.go:387] No image sha256:ab60623bb32f7e75fca71ef65137731cae347a21c7a4091dfd583fa00732721c (100ms) I0507 15:33:24.981935 1 gather_workloads_info.go:387] No image sha256:3bba1358d4a0ae878ff491c0c2cbfffe60649e110b40342b878fe8fa332f8858 (109ms) I0507 15:33:25.073020 1 gather_workloads_info.go:387] No image sha256:5d6a2b8d2508c7da71192661e7c0dcf1335f48b6336a20189abffbe79eeff962 (91ms) I0507 15:33:25.173286 1 gather_workloads_info.go:387] No image sha256:084aa9b0f8a6d478549dc384d4e66da13ee9b25cc98531da861cc19dee2a9e8f (100ms) I0507 15:33:25.273477 1 gather_workloads_info.go:387] No image sha256:934d8e8c50f3c609b8eea80d1051111fe3d066fe8c65c79572072ae55fcb0a86 (100ms) I0507 15:33:25.373494 1 gather_workloads_info.go:387] No image sha256:89277d8d4560d71db88c2dcc67c992a24544ca21810920b609c1d49d53b4a287 (100ms) I0507 15:33:25.473224 1 gather_workloads_info.go:387] No image sha256:bdeb1c455cffccdb6c90db238e9d371de06340b1d6105d47f66d87293f878ac1 (100ms) I0507 15:33:25.573098 1 gather_workloads_info.go:387] No image sha256:637d41f067a5239096fc22b135181cda5113da833f1370e7a73965e83792e93a (100ms) I0507 15:33:25.673149 1 gather_workloads_info.go:387] No image sha256:572ebf7c7fab1a518582f3ca46c866730423cce6e8a19f614dc15fd61bf0c65a (100ms) I0507 15:33:25.774299 1 gather_workloads_info.go:387] No image sha256:0acfe677ed58031f8381cb8a4f2d8917ec4cfa95485e2a46529dffae54c75ace (101ms) I0507 15:33:25.872603 1 gather_workloads_info.go:387] No image sha256:15e1b5f763553e86d0230a751b0783e3b990438fd07cf96eed164bccb72ed564 (98ms) I0507 15:33:25.872638 1 tasks_processing.go:74] worker 1 stopped. E0507 15:33:25.872647 1 gather.go:140] gatherer "workloads" function "workload_info" failed with the error: received unexpected status code 500 Internal Server Error from https://10.128.0.26:8443/gather_runtime_info I0507 15:33:25.872915 1 recorder.go:75] Recording config/workload_info with fingerprint=23f8cbb6bf9e529a0e1376202158ad173a749f541657e5e6728f3637c597a872 I0507 15:33:25.872931 1 gather.go:177] gatherer "workloads" function "workload_info" took 8.575555456s to process 1 records E0507 15:33:25.872957 1 periodic.go:254] "Unhandled Error" err="workloads failed after 8.575s with: function \"workload_info\" failed with an error" I0507 15:33:25.874074 1 controllerstatus.go:89] name=periodic-workloads healthy=false reason=PeriodicGatherFailed message=Source workloads could not be retrieved: function "workload_info" failed with an error I0507 15:33:25.874087 1 periodic.go:216] Running conditional gatherer I0507 15:33:25.881038 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.21.9/gathering_rules I0507 15:33:26.038879 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer I0507 15:33:27.090139 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0507 15:33:27.097358 1 conditional_gatherer.go:392] cluster version is '4.21.9' I0507 15:33:27.097390 1 tasks_processing.go:45] number of workers: 3 I0507 15:33:27.097403 1 tasks_processing.go:69] worker 2 listening for tasks. I0507 15:33:27.097408 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0507 15:33:27.097475 1 tasks_processing.go:69] worker 1 listening for tasks. I0507 15:33:27.097489 1 tasks_processing.go:71] worker 1 working on remote_configuration task. I0507 15:33:27.097494 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0507 15:33:27.097491 1 tasks_processing.go:69] worker 0 listening for tasks. I0507 15:33:27.097504 1 tasks_processing.go:74] worker 0 stopped. I0507 15:33:27.097514 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=c8d4892cd4f2eea27d57353c6055eeb530bc2d811bfe041f6984d90395255a8c I0507 15:33:27.097531 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.109µs to process 1 records I0507 15:33:27.097585 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0507 15:33:27.097598 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.12µs to process 1 records I0507 15:33:27.097606 1 tasks_processing.go:74] worker 1 stopped. I0507 15:33:27.097805 1 tasks_processing.go:74] worker 2 stopped. I0507 15:33:27.097816 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 305.193µs to process 0 records I0507 15:33:27.097829 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0507 15:33:27.097847 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0507 15:33:27.127964 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=eefefbc1804c4e1dd5b266f58507649a11a1f2c155726438dd4d4ec6a1ac31fa I0507 15:33:27.128107 1 diskrecorder.go:70] Writing 111 records to /var/lib/insights-operator/insights-2026-05-07-153327.tar.gz I0507 15:33:27.134787 1 diskrecorder.go:51] Wrote 111 records to disk in 6ms I0507 15:33:27.134820 1 periodic.go:285] Gathering cluster info every 2h0m0s I0507 15:33:27.134837 1 periodic.go:286] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0507 15:34:33.683939 1 diskrecorder.go:223] Found files to send: insights-2026-05-07-153327.tar.gz I0507 15:34:33.683970 1 insightsuploader.go:150] Checking archives to upload periodically every 16m7.070388067s I0507 15:34:33.683978 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0507 15:34:33.699196 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0507 15:34:34.034745 1 requests.go:87] Successfully reported id=2026-05-07T15:34:33Z x-rh-insights-request-id=509707885725460c8cc2178cbbfdc028, wrote=49881 I0507 15:34:34.034870 1 insightsuploader.go:187] Uploaded report successfully in 350.879155ms I0507 15:34:34.034907 1 controller.go:129] Initializing last reported time to 2026-05-07T15:34:33Z I0507 15:34:34.034984 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0507 15:34:34.034996 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0507 15:34:34.035005 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0507 15:34:34.042555 1 controller.go:554] The operator is healthy I0507 15:35:03.709690 1 controller.go:554] The operator is healthy I0507 15:35:40.010621 1 insightsreport.go:137] Pulling report from smart-proxy I0507 15:35:40.010647 1 insightsreport.go:149] Retrieving report I0507 15:35:40.018115 1 requests.go:110] Retrieving report for cluster: 3515d16e-f7aa-4d98-a80d-106de21e1eeb I0507 15:35:40.018128 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/3515d16e-f7aa-4d98-a80d-106de21e1eeb/reports I0507 15:35:40.021213 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/3515d16e-f7aa-4d98-a80d-106de21e1eeb/reports I0507 15:35:40.279718 1 insightsreport.go:184] Report retrieved I0507 15:35:40.309435 1 insightsreport.go:239] Report retrieved correctly I0507 15:37:03.709721 1 controller.go:554] The operator is healthy I0507 15:38:03.674370 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0507 15:38:03.681257 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0507 15:38:03.681277 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0507 15:38:03.686654 1 secretconfigobserver.go:119] support secret does not exist I0507 15:39:03.708457 1 controller.go:554] The operator is healthy I0507 15:41:03.710669 1 controller.go:554] The operator is healthy I0507 15:43:03.669605 1 configmapobserver.go:84] configmaps "insights-config" not found I0507 15:43:03.687741 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0507 15:43:03.693180 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0507 15:43:03.693197 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0507 15:43:03.701092 1 secretconfigobserver.go:119] support secret does not exist I0507 15:43:03.709238 1 controller.go:554] The operator is healthy