I0704 14:01:59.861349 1 cmd.go:253] Using service-serving-cert provided certificates I0704 14:01:59.861819 1 observer_polling.go:159] Starting file observer I0704 14:02:00.053762 1 operator.go:60] Starting insights-operator v0.0.0-master+$Format:%H$ I0704 14:02:00.053970 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0704 14:02:00.054560 1 secure_serving.go:57] Forcing use of http/1.1 only W0704 14:02:00.054582 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0704 14:02:00.054588 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0704 14:02:00.054594 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0704 14:02:00.054599 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0704 14:02:00.054602 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0704 14:02:00.054606 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0704 14:02:00.054603 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0704 14:02:00.058363 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"59c083d1-4182-4029-868e-f0359448c416", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}} I0704 14:02:00.058302 1 operator.go:125] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GatewayAPI GatewayAPIController HighlyAvailableArbiter ImageVolume IngressControllerLBSubnetsAWS KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController SigstoreImageVerification StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPClusterHostedDNSInstall GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall ImageModeStatusReporting ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesAzure ManagedBootImagesvSphere MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutatingAdmissionPolicy NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterOperations NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PreconfiguredUDNAddresses SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeAttributesClass VolumeGroupSnapshot] I0704 14:02:00.059737 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0704 14:02:00.059767 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0704 14:02:00.059808 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0704 14:02:00.059832 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0704 14:02:00.059861 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0704 14:02:00.059884 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0704 14:02:00.060351 1 secure_serving.go:213] Serving securely on [::]:8443 I0704 14:02:00.060418 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" I0704 14:02:00.060429 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/var/run/secrets/serving-cert/tls.crt::/var/run/secrets/serving-cert/tls.key" W0704 14:02:00.066461 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0704 14:02:00.066522 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0704 14:02:00.066610 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0704 14:02:00.076803 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:02:00.076828 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0704 14:02:00.085406 1 secretconfigobserver.go:119] support secret does not exist I0704 14:02:00.093360 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:02:00.101352 1 secretconfigobserver.go:119] support secret does not exist I0704 14:02:00.105540 1 recorder.go:161] Pruning old reports every 5h36m41s, max age is 288h0m0s I0704 14:02:00.113725 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0704 14:02:00.113746 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0704 14:02:00.113748 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0704 14:02:00.113758 1 insightsreport.go:296] Starting report retriever I0704 14:02:00.113766 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0704 14:02:00.113774 1 periodic.go:209] Running clusterconfig gatherer I0704 14:02:00.113833 1 tasks_processing.go:45] number of workers: 32 I0704 14:02:00.113855 1 tasks_processing.go:69] worker 1 listening for tasks. I0704 14:02:00.113866 1 tasks_processing.go:69] worker 11 listening for tasks. I0704 14:02:00.113867 1 tasks_processing.go:69] worker 0 listening for tasks. I0704 14:02:00.113871 1 tasks_processing.go:71] worker 11 working on sap_datahubs task. I0704 14:02:00.113869 1 tasks_processing.go:71] worker 1 working on image task. I0704 14:02:00.113881 1 tasks_processing.go:69] worker 23 listening for tasks. I0704 14:02:00.113887 1 tasks_processing.go:69] worker 24 listening for tasks. I0704 14:02:00.113891 1 tasks_processing.go:69] worker 21 listening for tasks. I0704 14:02:00.113896 1 tasks_processing.go:69] worker 25 listening for tasks. I0704 14:02:00.113898 1 tasks_processing.go:69] worker 6 listening for tasks. I0704 14:02:00.113905 1 tasks_processing.go:69] worker 26 listening for tasks. I0704 14:02:00.113902 1 tasks_processing.go:69] worker 12 listening for tasks. I0704 14:02:00.113909 1 tasks_processing.go:69] worker 17 listening for tasks. I0704 14:02:00.113913 1 tasks_processing.go:69] worker 2 listening for tasks. I0704 14:02:00.113916 1 tasks_processing.go:69] worker 27 listening for tasks. I0704 14:02:00.113918 1 tasks_processing.go:69] worker 18 listening for tasks. I0704 14:02:00.113922 1 tasks_processing.go:69] worker 16 listening for tasks. I0704 14:02:00.113926 1 tasks_processing.go:69] worker 3 listening for tasks. I0704 14:02:00.113924 1 tasks_processing.go:69] worker 28 listening for tasks. I0704 14:02:00.113932 1 tasks_processing.go:69] worker 20 listening for tasks. I0704 14:02:00.113935 1 tasks_processing.go:69] worker 13 listening for tasks. I0704 14:02:00.113926 1 tasks_processing.go:69] worker 19 listening for tasks. I0704 14:02:00.113940 1 tasks_processing.go:69] worker 5 listening for tasks. I0704 14:02:00.113941 1 tasks_processing.go:69] worker 30 listening for tasks. I0704 14:02:00.113943 1 tasks_processing.go:69] worker 14 listening for tasks. I0704 14:02:00.113947 1 tasks_processing.go:69] worker 31 listening for tasks. I0704 14:02:00.113950 1 tasks_processing.go:69] worker 15 listening for tasks. I0704 14:02:00.113933 1 tasks_processing.go:69] worker 29 listening for tasks. I0704 14:02:00.113951 1 tasks_processing.go:69] worker 8 listening for tasks. I0704 14:02:00.113956 1 tasks_processing.go:69] worker 7 listening for tasks. I0704 14:02:00.113954 1 tasks_processing.go:69] worker 4 listening for tasks. I0704 14:02:00.113959 1 tasks_processing.go:69] worker 9 listening for tasks. I0704 14:02:00.113961 1 tasks_processing.go:69] worker 10 listening for tasks. I0704 14:02:00.113875 1 tasks_processing.go:69] worker 22 listening for tasks. I0704 14:02:00.114034 1 tasks_processing.go:71] worker 6 working on dvo_metrics task. I0704 14:02:00.114039 1 tasks_processing.go:71] worker 26 working on machines task. I0704 14:02:00.114045 1 tasks_processing.go:71] worker 18 working on sap_pods task. I0704 14:02:00.114054 1 tasks_processing.go:71] worker 17 working on overlapping_namespace_uids task. I0704 14:02:00.114070 1 tasks_processing.go:71] worker 24 working on infrastructures task. I0704 14:02:00.114104 1 tasks_processing.go:71] worker 0 working on crds task. I0704 14:02:00.114161 1 tasks_processing.go:71] worker 29 working on tsdb_status task. I0704 14:02:00.114204 1 tasks_processing.go:71] worker 9 working on silenced_alerts task. I0704 14:02:00.114222 1 tasks_processing.go:71] worker 10 working on ingress_certificates task. I0704 14:02:00.114040 1 tasks_processing.go:71] worker 22 working on install_plans task. I0704 14:02:00.114111 1 tasks_processing.go:71] worker 2 working on nodes task. I0704 14:02:00.114121 1 tasks_processing.go:71] worker 14 working on machine_config_pools task. I0704 14:02:00.114125 1 tasks_processing.go:71] worker 16 working on monitoring_persistent_volumes task. I0704 14:02:00.114128 1 tasks_processing.go:71] worker 3 working on certificate_signing_requests task. I0704 14:02:00.114131 1 tasks_processing.go:71] worker 28 working on node_logs task. I0704 14:02:00.114121 1 tasks_processing.go:71] worker 23 working on olm_operators task. I0704 14:02:00.114134 1 tasks_processing.go:71] worker 20 working on support_secret task. I0704 14:02:00.114137 1 tasks_processing.go:71] worker 19 working on clusterroles task. I0704 14:02:00.114138 1 tasks_processing.go:71] worker 5 working on feature_gates task. I0704 14:02:00.114141 1 tasks_processing.go:71] worker 30 working on version task. I0704 14:02:00.114141 1 tasks_processing.go:71] worker 13 working on ceph_cluster task. I0704 14:02:00.114151 1 tasks_processing.go:71] worker 8 working on openshift_logging task. I0704 14:02:00.114151 1 tasks_processing.go:71] worker 4 working on container_runtime_configs task. I0704 14:02:00.114154 1 tasks_processing.go:71] worker 31 working on machine_sets task. I0704 14:02:00.114158 1 tasks_processing.go:71] worker 15 working on openstack_dataplanenodesets task. I0704 14:02:00.114155 1 tasks_processing.go:71] worker 7 working on ingress task. I0704 14:02:00.114164 1 tasks_processing.go:71] worker 12 working on operators task. I0704 14:02:00.114185 1 tasks_processing.go:71] worker 21 working on service_accounts task. I0704 14:02:00.114195 1 tasks_processing.go:71] worker 25 working on pdbs task. I0704 14:02:00.114035 1 tasks_processing.go:71] worker 27 working on openstack_version task. I0704 14:02:00.119782 1 tasks_processing.go:71] worker 11 working on lokistack task. I0704 14:02:00.119798 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 5.892678ms to process 0 records E0704 14:02:00.120767 1 gather_prometheus_tsdb_status.go:49] Unable to tsdb status: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.27.92:9091: connect: connection refused I0704 14:02:00.120792 1 tasks_processing.go:71] worker 29 working on metrics task. E0704 14:02:00.120807 1 gather.go:140] gatherer "clusterconfig" function "tsdb_status" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/api/v1/status/tsdb": dial tcp 172.30.27.92:9091: connect: connection refused I0704 14:02:00.120832 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 6.574577ms to process 0 records I0704 14:02:00.121368 1 tasks_processing.go:71] worker 1 working on sap_config task. I0704 14:02:00.121542 1 recorder.go:75] Recording config/image with fingerprint=269b932e8bbaa7991f75351cedd455030326e49574acb69f38d5f23a000d43bc I0704 14:02:00.121563 1 gather.go:177] gatherer "clusterconfig" function "image" took 7.477913ms to process 1 records E0704 14:02:00.123064 1 gather_most_recent_metrics.go:87] Unable to retrieve most recent metrics: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.27.92:9091: connect: connection refused I0704 14:02:00.123092 1 tasks_processing.go:71] worker 29 working on image_pruners task. E0704 14:02:00.123103 1 gather.go:140] gatherer "clusterconfig" function "metrics" failed with the error: Get "https://prometheus-k8s.openshift-monitoring.svc:9091/federate?match%5B%5D=cluster_installer&match%5B%5D=namespace%3Acontainer_cpu_usage%3Asum&match%5B%5D=namespace%3Acontainer_memory_usage_bytes%3Asum&match%5B%5D=vsphere_node_hw_version_total&match%5B%5D=virt_platform&match%5B%5D=console_helm_installs_total&match%5B%5D=console_helm_upgrades_total&match%5B%5D=console_helm_uninstalls_total&match%5B%5D=openshift_apps_deploymentconfigs_strategy_total&match%5B%5D=etcd_server_slow_apply_total&match%5B%5D=etcd_server_slow_read_indexes_total&match%5B%5D=haproxy_exporter_server_threshold": dial tcp 172.30.27.92:9091: connect: connection refused I0704 14:02:00.123145 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 2.277201ms to process 0 records I0704 14:02:00.139833 1 tasks_processing.go:71] worker 31 working on image_registries task. I0704 14:02:00.139837 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 21.832062ms to process 0 records I0704 14:02:00.139895 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 25.797366ms to process 0 records I0704 14:02:00.139904 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 22.20732ms to process 0 records I0704 14:02:00.139914 1 tasks_processing.go:71] worker 8 working on schedulers task. I0704 14:02:00.139946 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 21.859322ms to process 0 records I0704 14:02:00.139973 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 22.403587ms to process 0 records I0704 14:02:00.140194 1 tasks_processing.go:71] worker 18 working on proxies task. I0704 14:02:00.140207 1 tasks_processing.go:71] worker 24 working on authentication task. I0704 14:02:00.140216 1 tasks_processing.go:71] worker 15 working on cost_management_metrics_configs task. I0704 14:02:00.140228 1 tasks_processing.go:71] worker 13 working on jaegers task. I0704 14:02:00.140894 1 controller.go:128] Initializing last reported time to 0001-01-01T00:00:00Z I0704 14:02:00.140929 1 controller.go:326] The initial operator extension status is healthy I0704 14:02:00.140928 1 recorder.go:75] Recording config/infrastructure with fingerprint=bde810e7bad79c4a330682ce7076bf71cb4bb6e0ff4df9abb60ea3f1e311a7db I0704 14:02:00.140938 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0704 14:02:00.140942 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0704 14:02:00.140945 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0704 14:02:00.140946 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 26.123441ms to process 1 records I0704 14:02:00.140962 1 controller.go:489] The operator is still being initialized E0704 14:02:00.140964 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0704 14:02:00.140969 1 controller.go:512] The operator is healthy I0704 14:02:00.140971 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 23.403331ms to process 0 records I0704 14:02:00.140977 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 22.688991ms to process 0 records I0704 14:02:00.140983 1 tasks_processing.go:71] worker 4 working on cluster_apiserver task. I0704 14:02:00.141047 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s I0704 14:02:00.141108 1 tasks_processing.go:71] worker 20 working on openstack_controlplanes task. I0704 14:02:00.141121 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s W0704 14:02:00.141255 1 operator.go:288] started I0704 14:02:00.141323 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0704 14:02:00.142211 1 tasks_processing.go:71] worker 5 working on config_maps task. I0704 14:02:00.142401 1 recorder.go:75] Recording config/featuregate with fingerprint=fe8c9d44a2e1c84f0a2dabb77c04b063a324bb9c0495d3ee145c2ccc7b839975 I0704 14:02:00.142462 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 24.966535ms to process 1 records I0704 14:02:00.160331 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0704 14:02:00.160350 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0704 14:02:00.160335 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0704 14:02:00.167556 1 base_controller.go:82] Caches are synced for ConfigController I0704 14:02:00.167575 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0704 14:02:00.199080 1 tasks_processing.go:71] worker 27 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0704 14:02:00.199145 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 80.426079ms to process 0 records I0704 14:02:00.199359 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 79.422433ms to process 0 records I0704 14:02:00.199375 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 77.849532ms to process 0 records I0704 14:02:00.199381 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 82.760901ms to process 0 records E0704 14:02:00.199389 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0704 14:02:00.199397 1 gather.go:177] gatherer "clusterconfig" function "machines" took 85.25398ms to process 0 records I0704 14:02:00.199407 1 tasks_processing.go:71] worker 26 working on aggregated_monitoring_cr_names task. I0704 14:02:00.199639 1 tasks_processing.go:71] worker 28 working on nodenetworkconfigurationpolicies task. I0704 14:02:00.199646 1 tasks_processing.go:71] worker 1 working on oauths task. I0704 14:02:00.199769 1 tasks_processing.go:71] worker 11 working on operators_pods_and_events task. I0704 14:02:00.202740 1 tasks_processing.go:71] worker 15 working on container_images task. I0704 14:02:00.202789 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 62.265569ms to process 0 records W0704 14:02:00.203370 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0704 14:02:00.203483 1 tasks_processing.go:71] worker 14 working on mutating_webhook_configurations task. I0704 14:02:00.203569 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 88.004297ms to process 0 records I0704 14:02:00.203857 1 tasks_processing.go:71] worker 4 working on machine_autoscalers task. I0704 14:02:00.205069 1 recorder.go:75] Recording config/apiserver with fingerprint=2b35797434532bae32c3210acffa17e695571095699f7a9ac504c266431e9139 I0704 14:02:00.205153 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 62.865099ms to process 1 records I0704 14:02:00.205337 1 tasks_processing.go:71] worker 16 working on storage_cluster task. I0704 14:02:00.205948 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 89.571512ms to process 0 records I0704 14:02:00.206014 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 65.205304ms to process 0 records I0704 14:02:00.206047 1 tasks_processing.go:71] worker 13 working on openstack_dataplanedeployments task. I0704 14:02:00.206590 1 tasks_processing.go:71] worker 24 working on pod_network_connectivity_checks task. I0704 14:02:00.207425 1 recorder.go:75] Recording config/authentication with fingerprint=4f5d54fdb97da9a6e28bbabae95ef29c53b26f777876dbb117de367399bde236 I0704 14:02:00.207447 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 66.231845ms to process 1 records I0704 14:02:00.207566 1 recorder.go:75] Recording config/proxy with fingerprint=25d05e75d2eaf6c7de48881ede3fc512ee25212e5cab545d6fecf26b18759cda I0704 14:02:00.207584 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 66.689451ms to process 1 records I0704 14:02:00.207594 1 tasks_processing.go:71] worker 18 working on machine_configs task. I0704 14:02:00.207985 1 tasks_processing.go:71] worker 29 working on storage_classes task. I0704 14:02:00.208308 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=52da3125d44f5612ddfbe1299f840c0c54411e5d3be0bfa4568f4658ed0d71e0 I0704 14:02:00.208328 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 84.871608ms to process 1 records I0704 14:02:00.213646 1 tasks_processing.go:71] worker 2 working on active_alerts task. I0704 14:02:00.214154 1 recorder.go:75] Recording config/node/ip-10-0-0-251.ec2.internal with fingerprint=f19d2913cd1104da487522a36b8676df522ac1ae187f5ab85621c96bf55f7df7 I0704 14:02:00.214221 1 recorder.go:75] Recording config/node/ip-10-0-1-212.ec2.internal with fingerprint=7463deba2e196ebcd4a299c440763815d70e805f153e405da95171734cf3378f I0704 14:02:00.214278 1 recorder.go:75] Recording config/node/ip-10-0-2-141.ec2.internal with fingerprint=0cdb9ff757a247b035958dd33161a253ca9ba177a406ffdd8bd2f85c6816dca8 I0704 14:02:00.214287 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 98.718026ms to process 3 records I0704 14:02:00.214298 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 72.509675ms to process 0 records I0704 14:02:00.214317 1 tasks_processing.go:71] worker 20 working on nodenetworkstates task. I0704 14:02:00.214410 1 recorder.go:75] Recording config/ingress with fingerprint=104908c438dad43b12771d3dcf522bb20194602424cba940b043d9ecd16d806b I0704 14:02:00.214420 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 95.642183ms to process 1 records I0704 14:02:00.214427 1 tasks_processing.go:71] worker 7 working on networks task. I0704 14:02:00.214472 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=f7b20f47fe172c22a23ae1c5d01267f051b7c0979415add0389ce792bdaa9005 I0704 14:02:00.214480 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 73.975124ms to process 1 records I0704 14:02:00.214484 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 97.493935ms to process 0 records I0704 14:02:00.214577 1 tasks_processing.go:71] worker 8 working on machine_healthchecks task. I0704 14:02:00.214589 1 tasks_processing.go:71] worker 23 working on openshift_machine_api_events task. I0704 14:02:00.217674 1 tasks_processing.go:71] worker 0 working on qemu_kubevirt_launcher_logs task. I0704 14:02:00.218365 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=0892bd10a3bdc716ab6049449d6a382edc17f1f3466ea8d72a3385e69da0a48f I0704 14:02:00.218610 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=1e2921b639a6cee417b485ed5de26e9624a0efa233a9861c8b4239f47283296e I0704 14:02:00.218626 1 gather.go:177] gatherer "clusterconfig" function "crds" took 103.498049ms to process 2 records I0704 14:02:00.218656 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0704 14:02:00.218674 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 104.084079ms to process 1 records I0704 14:02:00.218684 1 tasks_processing.go:71] worker 17 working on validating_webhook_configurations task. I0704 14:02:00.220173 1 tasks_processing.go:74] worker 3 stopped. I0704 14:02:00.220196 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 104.018414ms to process 0 records I0704 14:02:00.222715 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0704 14:02:00.222728 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0704 14:02:00.222733 1 controller.go:212] Source scaController *sca.Controller is not ready I0704 14:02:00.222737 1 controller.go:212] Source clusterTransferController *clustertransfer.Controller is not ready I0704 14:02:00.222739 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0704 14:02:00.222759 1 controller.go:489] The operator is still being initialized I0704 14:02:00.222764 1 controller.go:512] The operator is healthy I0704 14:02:00.226457 1 tasks_processing.go:74] worker 25 stopped. I0704 14:02:00.226661 1 recorder.go:75] Recording config/pdbs/openshift-console/console with fingerprint=91e8796b1bb8539a749797b8ccbe18cd61568adf6056e4cc49bb23c143969896 I0704 14:02:00.226684 1 recorder.go:75] Recording config/pdbs/openshift-console/downloads with fingerprint=0e711527a77cd270d6d9c2444e66adf048df810b64516bed7584b0180725ea18 I0704 14:02:00.226702 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=151ce6ae844e5b9f523a694c7f1368c578331e00ff3c2933fd8002f051f90402 I0704 14:02:00.226718 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=34e64558a51defdf8f81b5d9209ca3c3bc2e12dd76191abeaa34eb1ab13776f5 I0704 14:02:00.226737 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/alertmanager-main with fingerprint=85ccb980ee7b408b9a0f7d42b8a532b26fdbaafa24e2c17e69fe60b30b249873 I0704 14:02:00.226752 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/metrics-server with fingerprint=d7166ea23b3b1065cabce57c53f0ff29538e78657599e553b52b0f1c283d6257 I0704 14:02:00.226768 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/monitoring-plugin with fingerprint=b8e83e5f46f1daaa262a79aac564e1c67e70acd845eeb94987b21b82c9e52d45 I0704 14:02:00.226787 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-k8s with fingerprint=eca25e864e32c1ade2aca3ff2c5c18dc1395d02261548b71c08d9c81cc105fd9 I0704 14:02:00.226804 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/prometheus-operator-admission-webhook with fingerprint=152de9a390e764570fe13bfcde82777f76797ad45ba138b87c64a676e865fc21 I0704 14:02:00.226821 1 recorder.go:75] Recording config/pdbs/openshift-monitoring/thanos-querier-pdb with fingerprint=465744d0805d489a75d60937772522ffc169693383b3bfc69567eefcad714ac5 I0704 14:02:00.226837 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=2a1b8268c7191595083dbc43df4064e1a258a16e2c64bd37db250ca5ea142c1a I0704 14:02:00.226844 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 107.986186ms to process 11 records I0704 14:02:00.230439 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 14:02:00.230568 1 tasks_processing.go:74] worker 19 stopped. I0704 14:02:00.231066 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=0b71562a05ce9d93ac363afbb40e0df61d75c148af123e45b424f27ef327c41d I0704 14:02:00.231233 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=817d08fee869a778a6bc1ed57ff4e5f8f464b62d2ccf8a06e7592187fd8abcef I0704 14:02:00.231286 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 113.535742ms to process 2 records I0704 14:02:00.231348 1 tasks_processing.go:74] worker 1 stopped. I0704 14:02:00.231897 1 recorder.go:75] Recording config/oauth with fingerprint=425a24494e1184b2eddfd6540e62fe1851a574945fce1ba3616509de22b2ee0b I0704 14:02:00.231970 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 31.686679ms to process 1 records I0704 14:02:00.232916 1 tasks_processing.go:74] worker 31 stopped. I0704 14:02:00.234004 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=b9a02e891c47007b5a4ba59ed8097cbad324580d86cf1121035040245330130d I0704 14:02:00.234025 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 93.051293ms to process 1 records I0704 14:02:00.236907 1 tasks_processing.go:74] worker 28 stopped. I0704 14:02:00.236924 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 37.250624ms to process 0 records I0704 14:02:00.241741 1 base_controller.go:82] Caches are synced for LoggingSyncer I0704 14:02:00.241760 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0704 14:02:00.247789 1 tasks_processing.go:74] worker 13 stopped. I0704 14:02:00.247837 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 41.710897ms to process 0 records I0704 14:02:00.247897 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 43.888012ms to process 0 records E0704 14:02:00.247909 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0704 14:02:00.247919 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 41.190698ms to process 0 records I0704 14:02:00.247926 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 42.488232ms to process 0 records I0704 14:02:00.247930 1 tasks_processing.go:74] worker 4 stopped. I0704 14:02:00.247937 1 tasks_processing.go:74] worker 16 stopped. I0704 14:02:00.247941 1 tasks_processing.go:74] worker 24 stopped. I0704 14:02:00.248232 1 tasks_processing.go:74] worker 20 stopped. I0704 14:02:00.248245 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 33.901717ms to process 0 records E0704 14:02:00.248259 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0704 14:02:00.248271 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 33.659369ms to process 0 records I0704 14:02:00.248282 1 tasks_processing.go:74] worker 8 stopped. I0704 14:02:00.248473 1 tasks_processing.go:74] worker 30 stopped. I0704 14:02:00.248923 1 recorder.go:75] Recording config/version with fingerprint=732f42a3765cfc9433c9c40a3dfef656c3d36b6d7c8f4582ebe6568175b89a0b I0704 14:02:00.248941 1 recorder.go:75] Recording config/id with fingerprint=b6496e31d8f7ae6f3245ad63083cd7f70214ba99fa611dbc6a4166b979f3535c I0704 14:02:00.248950 1 gather.go:177] gatherer "clusterconfig" function "version" took 131.099806ms to process 2 records I0704 14:02:00.270444 1 gather_cluster_operator_pods_and_events.go:121] Found 0 pods with 0 containers I0704 14:02:00.270471 1 tasks_processing.go:74] worker 11 stopped. I0704 14:02:00.270481 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 70.680385ms to process 0 records I0704 14:02:00.273009 1 tasks_processing.go:74] worker 9 stopped. I0704 14:02:00.273115 1 recorder.go:75] Recording config/silenced_alerts.json with fingerprint=37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570 I0704 14:02:00.273136 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 158.778712ms to process 1 records I0704 14:02:00.273208 1 tasks_processing.go:74] worker 2 stopped. I0704 14:02:00.273244 1 recorder.go:75] Recording config/alerts with fingerprint=5ce87c3e579917bcbba6399eb136ca6ccdc60902ae89559f6a3f55a838f2b2aa I0704 14:02:00.273255 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 59.539534ms to process 1 records I0704 14:02:00.281699 1 gather_logs.go:145] no pods in namespace were found I0704 14:02:00.281720 1 tasks_processing.go:74] worker 23 stopped. I0704 14:02:00.281733 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 67.113242ms to process 0 records I0704 14:02:00.281815 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 64.055461ms to process 0 records I0704 14:02:00.281827 1 tasks_processing.go:74] worker 0 stopped. I0704 14:02:00.281843 1 tasks_processing.go:74] worker 14 stopped. I0704 14:02:00.282130 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=b037c90f7c078a7e685c2858a0fff5843ca549da85803f9d6fdf382b532fd9eb I0704 14:02:00.282188 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=55e4f21c43d2dd2e40a5c3476bd702d9d753f516c15695af9504fd1adc54c061 I0704 14:02:00.282225 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=257ffd4f2e3a14032abbef639b2bb68a232d0528df777209b8163f848eaa42a9 I0704 14:02:00.282240 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 78.281819ms to process 3 records I0704 14:02:00.282337 1 tasks_processing.go:74] worker 7 stopped. I0704 14:02:00.282444 1 recorder.go:75] Recording config/network with fingerprint=6b3cd40709350f672f988ae41e1ef3f99b60ebddf480e7c251a8e2dc4c16bca4 I0704 14:02:00.282460 1 gather.go:177] gatherer "clusterconfig" function "networks" took 67.542815ms to process 1 records I0704 14:02:00.282521 1 tasks_processing.go:74] worker 29 stopped. I0704 14:02:00.282566 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=ce2c4df8bf1d32dece62d3421044233ce7cc591d0f1c0f2dc14bb2759f59f741 I0704 14:02:00.282585 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=7b99920522c09dd731910a6a785e09d8860b38cf80f7eb4fc42e214bfcff97d2 I0704 14:02:00.282591 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 74.395769ms to process 2 records I0704 14:02:00.284292 1 tasks_processing.go:74] worker 17 stopped. I0704 14:02:00.284568 1 recorder.go:75] Recording config/validatingwebhookconfigurations/alertmanagerconfigs.openshift.io with fingerprint=45f6f5a53e6d5ceb63dfc8f5df3fe654da968c00dd7c2630060ee07c5529bb9c I0704 14:02:00.284804 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=469ffd0e47e85e4b34380765b5361569b9f805a78eff01c9945afe8489a50805 I0704 14:02:00.285635 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=549e6e0564de5da2207e16bc18f40e50400165db6cdda1dc1a47565fec1a0f9a I0704 14:02:00.285755 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=30f5ad6648d78b9097a325170c89ed6e6546a16b20abbaa8832a521ee3aefc64 I0704 14:02:00.285819 1 recorder.go:75] Recording config/validatingwebhookconfigurations/prometheusrules.openshift.io with fingerprint=bc0d77bcd6b137d25351d3b250c9c48dffc2a057cbcdd15424df64299033b031 I0704 14:02:00.285895 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=056699721769c5380e5d274727d86ca750d55647a02a5493635cb86fd115f1d7 I0704 14:02:00.285968 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=98186a5ec26577fa75ca5e3937e52cc439ec5b209d72bebaa8fb6e4985a179d7 I0704 14:02:00.286035 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=87a1ece49c03798cf45e931305f76f873f340cf049fa3d9bfb09abb6219aeabb I0704 14:02:00.286092 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=1dbf526ac87f1143bdafd4a94c6b18c0b77d0a141479f89b332c7c7a0e49bb06 I0704 14:02:00.286176 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=dde18b1b833b0ec0ad13ba91775c3fc624b226ce9d4c86303c10b71b25715f8d I0704 14:02:00.286246 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=279b192444d5a72664cf6a5b19218ceeba7c758160f706d2fdaaca6a24842c31 I0704 14:02:00.286308 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=00a0f98835fe08820cdf10fddddb36aa8bbb885ef7809acb55296e0633d05cc7 I0704 14:02:00.286397 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=7cde01af6b504da719100e9367fcae5c62754a6bcea213828250e0cf9246fc4a I0704 14:02:00.286433 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 65.596787ms to process 13 records I0704 14:02:00.295415 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0704 14:02:00.312945 1 tasks_processing.go:74] worker 18 stopped. I0704 14:02:00.313021 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0704 14:02:00.313087 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 105.338083ms to process 1 records I0704 14:02:00.314820 1 tasks_processing.go:74] worker 26 stopped. I0704 14:02:00.314917 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 115.401739ms to process 0 records I0704 14:02:00.322195 1 tasks_processing.go:74] worker 15 stopped. I0704 14:02:00.323581 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-575cd97545-5g772 with fingerprint=637b2fba86d10f53cf7c217cf8d6d091ed7fcbad8c0c53bfd6b42a70a0c77fcb I0704 14:02:00.323681 1 recorder.go:75] Recording config/pod/openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-0-251.ec2.internal with fingerprint=11ccf0fac74be212318045fe09df8796720570093a018bc7956759e23129ae7b I0704 14:02:00.323752 1 recorder.go:75] Recording config/pod/openshift-machine-config-operator/kube-rbac-proxy-crio-ip-10-0-1-212.ec2.internal with fingerprint=13701274dee751f1c405ca5852ea2978a0c5924a89dada76a3ce70869eed0f46 I0704 14:02:00.324017 1 recorder.go:75] Recording config/pod/openshift-ovn-kubernetes/ovnkube-node-lsmvl with fingerprint=71d7baa9c549f143306144cac4d0e13c5eee1ff3cff46c2a243040ce7e222a49 I0704 14:02:00.324088 1 recorder.go:75] Recording config/running_containers with fingerprint=f8acc881c6d64ee0272bd3c44e07fa28029035b66bda5333c186c3e54a16b113 I0704 14:02:00.324098 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 119.425942ms to process 5 records I0704 14:02:00.338954 1 cluster_transfer.go:102] no available accepted cluster transfer I0704 14:02:00.338973 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=NoClusterTransfer message=no available cluster transfer I0704 14:02:00.340540 1 tasks_processing.go:74] worker 5 stopped. E0704 14:02:00.340557 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0704 14:02:00.340563 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0704 14:02:00.340567 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0704 14:02:00.340579 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=26b6661162b099a0f5a279859b4f46c867929a79d9a4a41fde4be4e6fe138018 I0704 14:02:00.340614 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=f73fcd48d6f9ebbc83253a1b5f23ef59d0683465ec46e474389103c717467dfb I0704 14:02:00.340622 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0704 14:02:00.340626 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=c93090eb0d2a4736885abeb79c91680cfd01fda46464f83456b085d4dc8239f0 I0704 14:02:00.340635 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=34cdf5d36d45a9e3ee7c2be31546e3d937ae4ebf144ac07b0210c06a1610a94d I0704 14:02:00.340672 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0704 14:02:00.340684 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0704 14:02:00.340690 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 198.261758ms to process 7 records I0704 14:02:00.344471 1 tasks_processing.go:74] worker 10 stopped. E0704 14:02:00.344526 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0704 14:02:00.344541 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2rbbsq8odit7endtrpi0nqhn4ru7u410-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2rbbsq8odit7endtrpi0nqhn4ru7u410-primary-cert-bundle-secret" not found I0704 14:02:00.344710 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=758e0eae19dfececcaec910db558f56cc7713ea433e24a078f04077a43a22255 I0704 14:02:00.344735 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 230.219194ms to process 1 records I0704 14:02:00.357249 1 tasks_processing.go:74] worker 27 stopped. I0704 14:02:00.357273 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 158.064183ms to process 0 records I0704 14:02:00.406017 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0704 14:02:00.416092 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0704 14:02:00.500945 1 sca.go:266] etc-pki-entitlement secret successfully created I0704 14:02:00.500968 1 controllerstatus.go:80] name=scaController healthy=true reason=Updated message=SCA certs successfully updated W0704 14:02:01.202209 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0704 14:02:02.201986 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0704 14:02:02.700483 1 tasks_processing.go:74] worker 12 stopped. I0704 14:02:02.700662 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=efcb10d400e0ff379aef59f14b133a651f33cc34c34257647e09c881bea83507 I0704 14:02:02.700723 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/console/cluster with fingerprint=be84fc75c87766f0f4d4e9fb1f9db618d008a21cc463e8239af81a1c296612d1 I0704 14:02:02.700750 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=04a42488db383dea495c57a552c0490fa5014bdb8608a84d45ee83b2cc22554b I0704 14:02:02.700759 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0704 14:02:02.700784 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=355c753aec432eb33ab7f2f7e994240bdae6feb7a5ce526df654ec679430a164 I0704 14:02:02.700800 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0704 14:02:02.700825 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=1d98b6c7c476d4c7434ab905198f5cad040a5cc77e5350a919b356d944fdd9cf I0704 14:02:02.700853 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=efccce9ad067bdfc9c35ce40c01bf75bd37e1d38102705193e237928ae66e5fb I0704 14:02:02.700881 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=a1ff06b2041c81e66dd86e258138f98330664493a4669f4357af5de771606d03 I0704 14:02:02.700911 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=92643f67c4856903544dc89b38a68d1d6e20f47afa4ce3707389745dd318b92c I0704 14:02:02.701410 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/insightsoperator/cluster with fingerprint=e82982f2602e7b887a94ce0cda56553a005f16a583933a9ed6d822d8e64009ee I0704 14:02:02.701460 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=ef24e31b04833072bd59f665f606e16938f1073ad7d3936e2b969a53bb25dd7d I0704 14:02:02.701475 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0704 14:02:02.701533 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=5ccf62ec4aef55af1026888fb6e9cc6375cdab33d47a0c02283b284981a51106 I0704 14:02:02.701552 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0704 14:02:02.701630 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=d17b1f75eb2434f76fedb6a28717c4a271c671be2658a23b5a65b638497ebfa8 I0704 14:02:02.701760 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0704 14:02:02.701827 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=d9f380590abf39a849817cdca6ebebfd7695ed0026a21713f89dbd4a8d5a4625 I0704 14:02:02.701853 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0704 14:02:02.701889 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=61c80b785f82c68f9c851f647ca586a1660f6afbb6ea3e3bccedf04192b2deaf I0704 14:02:02.702130 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=a8b8ea32274eee68bdad9bc19e4e7c1a6459d0dbf5796ed54fa938085b0964b5 I0704 14:02:02.702146 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0704 14:02:02.702159 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0704 14:02:02.702217 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0704 14:02:02.702261 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=467fd42168a4a949c4d04b29690e9247fe1f8457b23ce17c0910b6e0a614a0e1 I0704 14:02:02.702304 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=dcee2afeed2a0e59ef711a27fd58e63d756f8b1a8e037e3c4ceaced71bba004e I0704 14:02:02.702328 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0704 14:02:02.702357 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=d98af1334b9c168c812242a8ef61dcf16d903a93af3237559268b4b522a48888 I0704 14:02:02.702370 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0704 14:02:02.702406 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=6cd58bb6c7f3bab9c23ba06e0a2a6c7994ac3a155ef15363613640845a627a0d I0704 14:02:02.702420 1 recorder.go:75] Recording config/clusteroperator/samples.operator.openshift.io/config/cluster with fingerprint=439a4284281b8dcef0621bb14ba23e2175a28a50613f86b33a171c49689474fd I0704 14:02:02.702441 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=a7c79d1bc2bdd6fba9442d72a4e4f2eb233594b7213de787df9a5623a4f2f687 I0704 14:02:02.702540 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=a32d0cf18a7cfa94be03c75a3308f4330737b04839f7b287ece849ff63685fd3 I0704 14:02:02.702569 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=1b7efd2be33e3301a70dca97e715d19a8948b3b81fc510c0e79f03aabd177d18 I0704 14:02:02.702606 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=a8b04ab099c642b09a5313e2730fdd77f43b104e3368f05179926921b6d1f857 I0704 14:02:02.702626 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0704 14:02:02.702714 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=7143e2bbc75aad838d58b26a1e4383876db9ea20ae8a1a27c2349e85cf798e21 I0704 14:02:02.702752 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0704 14:02:02.702801 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0704 14:02:02.702816 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.582093408s to process 39 records W0704 14:02:03.202148 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0704 14:02:04.202098 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0704 14:02:05.201891 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0704 14:02:05.201924 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0704 14:02:05.201944 1 tasks_processing.go:74] worker 6 stopped. E0704 14:02:05.201955 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0704 14:02:05.201966 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0704 14:02:05.201976 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.087895238s to process 1 records I0704 14:02:13.223649 1 tasks_processing.go:74] worker 22 stopped. I0704 14:02:13.223689 1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b I0704 14:02:13.223701 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 13.109273628s to process 1 records I0704 14:02:13.924758 1 tasks_processing.go:74] worker 21 stopped. I0704 14:02:13.925039 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=b5628417db3123c06333720ebefb8c69f4980e1afaf323f1ff9cb97dfe717013 I0704 14:02:13.925058 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.806322811s to process 1 records E0704 14:02:13.925134 1 periodic.go:247] "Unhandled Error" err="clusterconfig failed after 13.811s with: function \"tsdb_status\" failed with an error, function \"metrics\" failed with an error, function \"support_secret\" failed with an error, function \"machines\" failed with an error, function \"pod_network_connectivity_checks\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0704 14:02:13.926262 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "tsdb_status" failed with an error, function "metrics" failed with an error, function "support_secret" failed with an error, function "machines" failed with an error, function "pod_network_connectivity_checks" failed with an error, function "machine_healthchecks" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0704 14:02:13.926279 1 periodic.go:209] Running workloads gatherer I0704 14:02:13.926294 1 tasks_processing.go:45] number of workers: 2 I0704 14:02:13.926300 1 tasks_processing.go:69] worker 1 listening for tasks. I0704 14:02:13.926304 1 tasks_processing.go:71] worker 1 working on workload_info task. I0704 14:02:13.926313 1 tasks_processing.go:69] worker 0 listening for tasks. I0704 14:02:13.926323 1 tasks_processing.go:71] worker 0 working on helmchart_info task. I0704 14:02:13.932329 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-1-212.ec2.internal... I0704 14:02:13.932350 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-0-251.ec2.internal... I0704 14:02:13.932334 1 gather_workloads_runtime_infos.go:63] Gathering workload runtime info for node ip-10-0-2-141.ec2.internal... I0704 14:02:13.951220 1 tasks_processing.go:74] worker 0 stopped. I0704 14:02:13.951242 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 24.881272ms to process 0 records I0704 14:02:20.033926 1 gather_workloads_runtime_infos.go:84] Gathered workload runtime infos in 6s I0704 14:02:20.062980 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 25s for image data I0704 14:02:20.071291 1 gather_workloads_info.go:387] No image sha256:04c87c054a3f366a7dfbe0a93ebb0c80a098ee16842c5794b67c1202eec61996 (10ms) I0704 14:02:20.080259 1 gather_workloads_info.go:387] No image sha256:765f0d23b637f685f98a31bd47c131b03cf72a40761a3f9a9d6320faa3c33733 (9ms) I0704 14:02:20.089004 1 gather_workloads_info.go:387] No image sha256:2904a78e2eb73fd6a9bb94c105c2a056831fb4113fbb7b0607c50adc9d879c9b (9ms) I0704 14:02:20.097626 1 gather_workloads_info.go:387] No image sha256:101993f6e65d6fda8b90c583b718b720809182961f69bc777295f47e6df4d7ce (9ms) I0704 14:02:20.106386 1 gather_workloads_info.go:387] No image sha256:5a95c19d82767e0235b4edb4a0536482c816904897aae1dc3eb255cb52b87a9f (9ms) I0704 14:02:20.114986 1 gather_workloads_info.go:387] No image sha256:56a85660a445eced5c79a595a0eccf590087c5672d50f49d4c25ad52f9a44f04 (9ms) I0704 14:02:20.124122 1 gather_workloads_info.go:387] No image sha256:a498046d64605bcccee2440aa4f04a4602baaae263cf01d977ec5208e876b1fd (9ms) I0704 14:02:20.133538 1 gather_workloads_info.go:387] No image sha256:b99613494e9d6a4a8207323872d63cc02b830fc80a8bf972e62c64381aa4cd17 (9ms) I0704 14:02:20.142780 1 gather_workloads_info.go:387] No image sha256:1a2532940843248c57d52141185dd71fbc393ab28b65d48f682038632c1dbbad (9ms) I0704 14:02:20.151476 1 gather_workloads_info.go:387] No image sha256:50197f22710766515f67944a779e00dd9ae3d17b18732d7324a970353b11f292 (9ms) I0704 14:02:20.170879 1 gather_workloads_info.go:398] Found image sha256:b3909bf664c77097f75b3768830863d642eed3815dab2bfb4415c771ca2d5007 (19ms) I0704 14:02:20.274960 1 gather_workloads_info.go:398] Found image sha256:32d4561d4bf1b3c4e3a7878f918f9c1422bab85bdf8711fbdf905d59692fee81 (104ms) I0704 14:02:20.372199 1 gather_workloads_info.go:387] No image sha256:289816958633a763a72dbc44e1dad40466223164e7e253039514f0d974ea5d21 (97ms) I0704 14:02:20.474673 1 gather_workloads_info.go:387] No image sha256:91828234f107c068c8a4966d08370ae7b73e637651dbc6d92c18c4553402c22c (102ms) I0704 14:02:20.572333 1 gather_workloads_info.go:387] No image sha256:3d6ea9e1633f38070718a1357fcb80e6cb3932aafcc4d65e659795b186c5da15 (98ms) I0704 14:02:20.671603 1 gather_workloads_info.go:387] No image sha256:7adc1eab05d6724c76ba751f6df816b08d6e70b78dee9eb94fa6fd9690542c98 (99ms) I0704 14:02:20.772698 1 gather_workloads_info.go:387] No image sha256:36b9e89c3cfcf1ab9ae500486e38afb6862cba48cb0b4d84a09508ab8f3d299f (101ms) I0704 14:02:20.875754 1 gather_workloads_info.go:387] No image sha256:09e27b3a67ebeda04d3e597e0561bfb40550de78bdecb53f2f9ff3dfb1cfe7ea (103ms) I0704 14:02:20.972196 1 gather_workloads_info.go:387] No image sha256:2e57e192c3c1240fd935dcd55c8fde5e70e78bf81d6176c96edf21fafe59f8ba (96ms) I0704 14:02:21.071571 1 gather_workloads_info.go:387] No image sha256:943018739e3db1763c3184b460dbc409e058abbac76d57b9927faad317be85e4 (99ms) I0704 14:02:21.171748 1 gather_workloads_info.go:387] No image sha256:3612c5a433effd17b7aff7ff6edf4fb27b61153908e3a0294158e4886a1d600d (100ms) I0704 14:02:21.272295 1 gather_workloads_info.go:387] No image sha256:a258c226562adb14e3a163a1940938526ee6a0928982a7667d85d9a7334ce639 (101ms) I0704 14:02:21.371901 1 gather_workloads_info.go:387] No image sha256:2598489729a4b258e4ecda4a06f6875133f2a10ced5c5241f8a57a8a05418e36 (100ms) I0704 14:02:21.471992 1 gather_workloads_info.go:387] No image sha256:7bd3e88d6d3f878a89203df590b684c5a77bf08df4717451d3197ab74683a80e (100ms) I0704 14:02:21.572112 1 gather_workloads_info.go:387] No image sha256:0af73ad7a47c9ff53269ae2bc5163bf9e389085220c34e4bd42c92c4e7a85f6d (100ms) I0704 14:02:21.671928 1 gather_workloads_info.go:387] No image sha256:c940ea87e7d133d75ba0002ef00c0806825eed3db8094cdb260d1bac18127733 (100ms) I0704 14:02:21.772118 1 gather_workloads_info.go:387] No image sha256:95418d0ba6fc0aa96647d97cf6530967f7d7ae97a82ce7f7e41c8e44310b0b72 (100ms) I0704 14:02:21.871850 1 gather_workloads_info.go:387] No image sha256:25bda9d34e23ba6bc9df5cf7104bdb237269740a3a76ee025f88b34e68b7e2b5 (100ms) I0704 14:02:21.971876 1 gather_workloads_info.go:387] No image sha256:653c666f842c13e0baae2e89a9b1efe0e2ef56f621ffb5b32005115d2a26ab8c (100ms) I0704 14:02:22.076079 1 gather_workloads_info.go:387] No image sha256:6e5df2aab9ce759e8f957341a11c087c233307e3c8f8760d21270efc19ccd3ce (104ms) I0704 14:02:22.176132 1 gather_workloads_info.go:387] No image sha256:0092c6ff139f8698c91349d507eb3182bc5d6c4ae040f672e347ff504accdea2 (100ms) I0704 14:02:22.272247 1 gather_workloads_info.go:387] No image sha256:5f0b67cfbbc381243fb91ccc17345b56d05f4d717c667e8c644e5bf05633ba71 (96ms) I0704 14:02:22.371003 1 gather_workloads_info.go:387] No image sha256:47154813651033d59751fb655a384dbffb64dd26f10bd7f3be0c3128d0486356 (99ms) I0704 14:02:22.472027 1 gather_workloads_info.go:387] No image sha256:a56211d075aa43cbb491f669a5b2e46ee023dc95b7d51dbac28f463948c5ad61 (101ms) I0704 14:02:22.571723 1 gather_workloads_info.go:387] No image sha256:03cf4cd7ef1518610c6c7b3ad27d1622d82e98e3dc6e3f8e5d0fceb5c8d3786e (100ms) I0704 14:02:22.671782 1 gather_workloads_info.go:387] No image sha256:513b1166e376feca472fdd5174f6abf0abf02925d6a21b7b4d89a2ff2fc67bed (100ms) I0704 14:02:22.771856 1 gather_workloads_info.go:387] No image sha256:695cf2f0cc07683c2a3ce1eaf3e56fe18abc6e2bac716f7d9843f5d173b9df52 (100ms) I0704 14:02:22.872201 1 gather_workloads_info.go:387] No image sha256:a0105d1eb62cf6ac9e5e2ef28d3e89bf6dc514bc594fc7090fe5a5ee18a09c87 (100ms) I0704 14:02:22.973387 1 gather_workloads_info.go:387] No image sha256:2e564f336c77116053f34d4201d364d8da04e789cfffa0ea422574c95f2d6404 (101ms) I0704 14:02:23.073811 1 gather_workloads_info.go:387] No image sha256:0a99240166165eb5718e7516a43282fe32df9c7c5e809b31b58abe44e42ff94d (100ms) I0704 14:02:23.172689 1 gather_workloads_info.go:387] No image sha256:76cc47f3775560477ec548c7b6fddbd99112ad6604f790697eef546576051fb7 (99ms) I0704 14:02:23.272099 1 gather_workloads_info.go:387] No image sha256:1e65f9f5490fbc930d2dce96f03ddcb28990b839d7dc011ef5de5bec6c5095bf (99ms) I0704 14:02:23.372389 1 gather_workloads_info.go:387] No image sha256:90bdf9e678fe18e35401e34e1cb0912d822e9cb32b9e41cc545870540e9a87f2 (100ms) I0704 14:02:23.471674 1 gather_workloads_info.go:387] No image sha256:7b31223098f08328f5ddea8e5b871dbbd5f5a61ec550e8956f66793c0c6031a9 (99ms) I0704 14:02:23.572582 1 gather_workloads_info.go:387] No image sha256:1477f3287cc75352305274f248f781c1b7c2a8696f4f7bc450c45a4a5f5b10ea (101ms) I0704 14:02:23.675093 1 gather_workloads_info.go:387] No image sha256:74713675d580a47a56b628469c4c05c41e114e0b935960d718ff0cf7e72f613d (102ms) I0704 14:02:23.772274 1 gather_workloads_info.go:387] No image sha256:4556896f77307821531ef91b7b7faccb82b824ea695693b2989f597f0deca038 (97ms) I0704 14:02:23.871162 1 gather_workloads_info.go:387] No image sha256:521712486e2c6e3c020dad6a1cb340db8e55665b69f7c208fab9cd9e965fd588 (99ms) I0704 14:02:23.871201 1 tasks_processing.go:74] worker 1 stopped. E0704 14:02:23.871213 1 gather.go:140] gatherer "workloads" function "workload_info" failed with the error: received unexpected status code 500 Internal Server Error from https://10.128.0.12:8443/gather_runtime_info I0704 14:02:23.871520 1 recorder.go:75] Recording config/workload_info with fingerprint=151bb3819659eb055c083882ea06770d286f05bbfc8ae1dcf155169eae915f40 I0704 14:02:23.871536 1 gather.go:177] gatherer "workloads" function "workload_info" took 9.944887586s to process 1 records E0704 14:02:23.871562 1 periodic.go:247] "Unhandled Error" err="workloads failed after 9.945s with: function \"workload_info\" failed with an error" I0704 14:02:23.872664 1 controllerstatus.go:89] name=periodic-workloads healthy=false reason=PeriodicGatherFailed message=Source workloads could not be retrieved: function "workload_info" failed with an error I0704 14:02:23.872680 1 periodic.go:209] Running conditional gatherer I0704 14:02:23.878696 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules I0704 14:02:24.084970 1 conditional_gatherer.go:338] updating alerts cache for conditional gatherer I0704 14:02:25.152404 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0704 14:02:25.159460 1 conditional_gatherer.go:392] cluster version is '4.20.8' I0704 14:02:25.159513 1 tasks_processing.go:45] number of workers: 3 I0704 14:02:25.159525 1 tasks_processing.go:69] worker 2 listening for tasks. I0704 14:02:25.159529 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0704 14:02:25.159537 1 tasks_processing.go:69] worker 0 listening for tasks. I0704 14:02:25.159551 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0704 14:02:25.159551 1 tasks_processing.go:69] worker 1 listening for tasks. I0704 14:02:25.159556 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0704 14:02:25.159561 1 tasks_processing.go:74] worker 1 stopped. I0704 14:02:25.159617 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=c8d4892cd4f2eea27d57353c6055eeb530bc2d811bfe041f6984d90395255a8c I0704 14:02:25.159630 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.108µs to process 1 records I0704 14:02:25.159667 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=5f681cc1442447fd2c1534f876836dff54caf3c545b582a5a8aae894f9203c5d I0704 14:02:25.159675 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 1.373µs to process 1 records I0704 14:02:25.159681 1 tasks_processing.go:74] worker 0 stopped. I0704 14:02:25.159824 1 tasks_processing.go:74] worker 2 stopped. I0704 14:02:25.159846 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 261.092µs to process 0 records I0704 14:02:25.159879 1 controllerstatus.go:80] name=periodic-conditional healthy=true reason=Succeeded message= I0704 14:02:25.159906 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=363fa5c8fa2573b18bc685faeca9ca9f021a8f80760895b7701f54e3dcd5c85f I0704 14:02:25.192046 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=4ab1946f2aac2be1b5f98a80f3268a5105c557df33fc377b0055b20bf9f73b4c I0704 14:02:25.192476 1 diskrecorder.go:70] Writing 113 records to /var/lib/insights-operator/insights-2026-07-04-140225.tar.gz I0704 14:02:25.200408 1 diskrecorder.go:51] Wrote 113 records to disk in 8ms I0704 14:02:25.200460 1 periodic.go:278] Gathering cluster info every 2h0m0s I0704 14:02:25.200516 1 periodic.go:279] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0704 14:03:30.114360 1 diskrecorder.go:223] Found files to send: insights-2026-07-04-140225.tar.gz I0704 14:03:30.114390 1 insightsuploader.go:150] Checking archives to upload periodically every 15m34.848433107s I0704 14:03:30.114399 1 insightsuploader.go:165] Uploading latest report since 0001-01-01T00:00:00Z I0704 14:03:30.128133 1 requests.go:46] Uploading application/vnd.redhat.openshift.periodic to https://console.redhat.com/api/ingress/v1/upload I0704 14:03:30.332447 1 requests.go:87] Successfully reported id=2026-07-04T14:03:30Z x-rh-insights-request-id=401a27da7ca04c938728af1f1281570b, wrote=54718 I0704 14:03:30.332605 1 insightsuploader.go:187] Uploaded report successfully in 218.196195ms I0704 14:03:30.332637 1 controller.go:128] Initializing last reported time to 2026-07-04T14:03:30Z I0704 14:03:30.332672 1 insightsreport.go:304] Archive uploaded, starting pulling report... I0704 14:03:30.332687 1 insightsreport.go:215] Starting retrieving report from Smart Proxy I0704 14:03:30.332699 1 insightsreport.go:221] Initial delay for pulling: 1m0s I0704 14:03:30.338400 1 controller.go:512] The operator is healthy I0704 14:04:00.148486 1 controller.go:512] The operator is healthy I0704 14:04:31.735350 1 insightsreport.go:137] Pulling report from smart-proxy I0704 14:04:31.735378 1 insightsreport.go:149] Retrieving report I0704 14:04:31.742090 1 requests.go:110] Retrieving report for cluster: ab3eadf9-defa-45b7-96fb-1a2463cdf27a I0704 14:04:31.742104 1 requests.go:111] Endpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/ab3eadf9-defa-45b7-96fb-1a2463cdf27a/reports I0704 14:04:31.744972 1 requests.go:121] Retrieving report from https://console.redhat.com/api/insights-results-aggregator/v2/cluster/ab3eadf9-defa-45b7-96fb-1a2463cdf27a/reports I0704 14:04:31.943231 1 insightsreport.go:184] Report retrieved I0704 14:04:31.979760 1 insightsreport.go:239] Report retrieved correctly I0704 14:06:00.147764 1 controller.go:512] The operator is healthy I0704 14:07:00.101908 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 14:07:00.143663 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:07:00.143709 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 14:07:00.159371 1 secretconfigobserver.go:119] support secret does not exist I0704 14:08:00.149660 1 controller.go:512] The operator is healthy I0704 14:10:00.148340 1 controller.go:512] The operator is healthy I0704 14:12:00.096456 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 14:12:00.100792 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 14:12:00.149714 1 controller.go:512] The operator is healthy I0704 14:12:00.160446 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 14:12:00.164935 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:12:00.164956 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 14:12:00.168726 1 secretconfigobserver.go:119] support secret does not exist I0704 14:14:00.148665 1 controller.go:512] The operator is healthy I0704 14:16:00.149966 1 controller.go:512] The operator is healthy I0704 14:17:00.169279 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 14:17:00.174559 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:17:00.174585 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 14:17:00.178397 1 secretconfigobserver.go:119] support secret does not exist I0704 14:18:00.146608 1 controller.go:512] The operator is healthy I0704 14:19:10.181423 1 insightsuploader.go:146] Nothing to report since 2026-07-04T14:03:30Z I0704 14:20:00.150237 1 controller.go:512] The operator is healthy I0704 14:22:00.101069 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 14:22:00.147598 1 controller.go:512] The operator is healthy I0704 14:22:00.179351 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 14:22:00.183698 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:22:00.183719 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 14:22:00.187198 1 secretconfigobserver.go:119] support secret does not exist I0704 14:24:00.150986 1 controller.go:512] The operator is healthy I0704 14:26:00.151599 1 controller.go:512] The operator is healthy I0704 14:27:00.187993 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 14:27:00.193476 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:27:00.193536 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 14:27:00.199179 1 secretconfigobserver.go:119] support secret does not exist I0704 14:28:00.147976 1 controller.go:512] The operator is healthy I0704 14:30:00.148247 1 controller.go:512] The operator is healthy I0704 14:32:00.098297 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 14:32:00.102362 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 14:32:00.146752 1 controller.go:512] The operator is healthy I0704 14:32:00.199625 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 14:32:00.205317 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:32:00.205341 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 14:32:00.208870 1 secretconfigobserver.go:119] support secret does not exist I0704 14:34:00.148135 1 controller.go:512] The operator is healthy I0704 14:34:16.900560 1 insightsuploader.go:146] Nothing to report since 2026-07-04T14:03:30Z I0704 14:36:00.150835 1 controller.go:512] The operator is healthy I0704 14:37:00.209739 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 14:37:00.218807 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:37:00.218835 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 14:37:00.223160 1 secretconfigobserver.go:119] support secret does not exist I0704 14:38:00.148011 1 controller.go:512] The operator is healthy I0704 14:40:00.147199 1 controller.go:512] The operator is healthy I0704 14:42:00.101382 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 14:42:00.147249 1 controller.go:512] The operator is healthy I0704 14:42:00.224208 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 14:42:00.229137 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:42:00.229162 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 14:42:00.233563 1 secretconfigobserver.go:119] support secret does not exist I0704 14:44:00.150898 1 controller.go:512] The operator is healthy I0704 14:46:00.148083 1 controller.go:512] The operator is healthy I0704 14:47:00.234188 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 14:47:00.239144 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:47:00.239175 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 14:47:00.242718 1 secretconfigobserver.go:119] support secret does not exist I0704 14:48:00.148160 1 controller.go:512] The operator is healthy I0704 14:50:00.152298 1 controller.go:512] The operator is healthy I0704 14:50:41.188465 1 insightsuploader.go:146] Nothing to report since 2026-07-04T14:03:30Z I0704 14:52:00.101304 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 14:52:00.151285 1 controller.go:512] The operator is healthy I0704 14:52:00.243316 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 14:52:00.247841 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:52:00.247867 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 14:52:00.251467 1 secretconfigobserver.go:119] support secret does not exist I0704 14:54:00.165354 1 controller.go:512] The operator is healthy I0704 14:56:00.151203 1 controller.go:512] The operator is healthy I0704 14:57:00.252536 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 14:57:00.260851 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 14:57:00.260873 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 14:57:00.264321 1 secretconfigobserver.go:119] support secret does not exist I0704 14:58:00.152839 1 controller.go:512] The operator is healthy I0704 15:00:00.149423 1 controller.go:512] The operator is healthy I0704 15:02:00.102635 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 15:02:00.147646 1 controller.go:512] The operator is healthy I0704 15:02:00.264852 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 15:02:00.269323 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 15:02:00.269346 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 15:02:00.273366 1 secretconfigobserver.go:119] support secret does not exist I0704 15:04:00.151431 1 controller.go:512] The operator is healthy I0704 15:06:00.150798 1 controller.go:512] The operator is healthy I0704 15:06:46.064467 1 insightsuploader.go:146] Nothing to report since 2026-07-04T14:03:30Z I0704 15:07:00.274019 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 15:07:00.278262 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 15:07:00.278290 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 15:07:00.282076 1 secretconfigobserver.go:119] support secret does not exist I0704 15:08:00.150821 1 controller.go:512] The operator is healthy I0704 15:10:00.149360 1 controller.go:512] The operator is healthy I0704 15:12:00.101764 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 15:12:00.105416 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 15:12:00.146772 1 controller.go:512] The operator is healthy I0704 15:12:00.282710 1 secretconfigobserver.go:136] Refreshing configuration from cluster pull secret I0704 15:12:00.287545 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 15:12:00.287567 1 secretconfigobserver.go:162] Refreshing configuration from cluster support secret I0704 15:12:00.291090 1 secretconfigobserver.go:119] support secret does not exist