W0704 06:58:30.346411 1 cmd.go:257] Using insecure, self-signed certificates I0704 06:58:31.576220 1 start.go:138] Unable to read service ca bundle: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0704 06:58:31.576563 1 observer_polling.go:159] Starting file observer I0704 06:58:32.221793 1 operator.go:60] Starting insights-operator v0.0.0-master+$Format:%H$ I0704 06:58:32.222040 1 legacy_config.go:327] Current config: {"report":false,"storagePath":"/var/lib/insights-operator","interval":"2h","endpoint":"https://console.redhat.com/api/ingress/v1/upload","conditionalGathererEndpoint":"https://console.redhat.com/api/gathering/v2/%s/gathering_rules","pull_report":{"endpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports","delay":"60s","timeout":"3000s","min_retry":"30s"},"impersonate":"system:serviceaccount:openshift-insights:gather","enableGlobalObfuscation":false,"ocm":{"scaEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates","scaInterval":"8h","scaDisabled":false,"clusterTransferEndpoint":"https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/","clusterTransferInterval":"12h"},"disableInsightsAlerts":false,"processingStatusEndpoint":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status","reportEndpointTechPreview":"https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/report"} I0704 06:58:32.222566 1 secure_serving.go:57] Forcing use of http/1.1 only W0704 06:58:32.222584 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256' detected. W0704 06:58:32.222588 1 secure_serving.go:69] Use of insecure cipher 'TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256' detected. W0704 06:58:32.222592 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_GCM_SHA256' detected. W0704 06:58:32.222596 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_GCM_SHA384' detected. W0704 06:58:32.222599 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_128_CBC_SHA' detected. W0704 06:58:32.222601 1 secure_serving.go:69] Use of insecure cipher 'TLS_RSA_WITH_AES_256_CBC_SHA' detected. I0704 06:58:32.222606 1 simple_featuregate_reader.go:171] Starting feature-gate-detector I0704 06:58:32.227154 1 event.go:377] Event(v1.ObjectReference{Kind:"Deployment", Namespace:"openshift-insights", Name:"insights-operator", UID:"87919429-7a27-47f2-b082-fe01a9c29844", APIVersion:"apps/v1", ResourceVersion:"", FieldPath:""}): type: 'Normal' reason: 'FeatureGatesInitialized' FeatureGates updated to featuregates.Features{Enabled:[]v1.FeatureGateName{"AdditionalRoutingCapabilities", "AdminNetworkPolicy", "AlibabaPlatform", "AzureWorkloadIdentity", "BuildCSIVolumes", "CPMSMachineNamePrefix", "ConsolePluginContentSecurityPolicy", "ExternalOIDC", "ExternalOIDCWithUIDAndExtraClaimMappings", "GatewayAPI", "GatewayAPIController", "HighlyAvailableArbiter", "ImageVolume", "IngressControllerLBSubnetsAWS", "KMSv1", "MachineConfigNodes", "ManagedBootImages", "ManagedBootImagesAWS", "MetricsCollectionProfiles", "NetworkDiagnosticsConfig", "NetworkLiveMigration", "NetworkSegmentation", "PinnedImages", "ProcMountType", "RouteAdvertisements", "RouteExternalCertificate", "ServiceAccountTokenNodeBinding", "SetEIPForNLBIngressController", "SigstoreImageVerification", "StoragePerformantSecurityPolicy", "UpgradeStatus", "UserNamespacesPodSecurityStandards", "UserNamespacesSupport", "VSphereMultiDisk", "VSphereMultiNetworks"}, Disabled:[]v1.FeatureGateName{"AWSClusterHostedDNS", "AWSClusterHostedDNSInstall", "AWSDedicatedHosts", "AWSServiceLBNetworkSecurityGroup", "AutomatedEtcdBackup", "AzureClusterHostedDNSInstall", "AzureDedicatedHosts", "AzureMultiDisk", "BootImageSkewEnforcement", "BootcNodeManagement", "ClusterAPIInstall", "ClusterAPIInstallIBMCloud", "ClusterMonitoringConfig", "ClusterVersionOperatorConfiguration", "DNSNameResolver", "DualReplica", "DyanmicServiceEndpointIBMCloud", "DynamicResourceAllocation", "EtcdBackendQuota", "EventedPLEG", "Example", "Example2", "ExternalSnapshotMetadata", "GCPClusterHostedDNS", "GCPClusterHostedDNSInstall", "GCPCustomAPIEndpoints", "GCPCustomAPIEndpointsInstall", "ImageModeStatusReporting", "ImageStreamImportMode", "IngressControllerDynamicConfigurationManager", "InsightsConfig", "InsightsConfigAPI", "InsightsOnDemandDataGather", "IrreconcilableMachineConfig", "KMSEncryptionProvider", "MachineAPIMigration", "MachineAPIOperatorDisableMachineHealthCheckController", "ManagedBootImagesAzure", "ManagedBootImagesvSphere", "MaxUnavailableStatefulSet", "MinimumKubeletVersion", "MixedCPUsAllocation", "MultiArchInstallAzure", "MultiDiskSetup", "MutatingAdmissionPolicy", "NewOLM", "NewOLMCatalogdAPIV1Metas", "NewOLMOwnSingleNamespace", "NewOLMPreflightPermissionChecks", "NewOLMWebhookProviderOpenshiftServiceCA", "NoRegistryClusterOperations", "NodeSwap", "NutanixMultiSubnets", "OVNObservability", "OpenShiftPodSecurityAdmission", "PreconfiguredUDNAddresses", "SELinuxMount", "ShortCertRotation", "SignatureStores", "SigstoreImageVerificationPKI", "TranslateStreamCloseWebsocketRequests", "VSphereConfigurableMaxAllowedBlockVolumesPerNode", "VSphereHostVMGroupZonal", "VSphereMixedNodeEnv", "VolumeAttributesClass", "VolumeGroupSnapshot"}} I0704 06:58:32.227018 1 operator.go:125] FeatureGates initialized: knownFeatureGates=[AdditionalRoutingCapabilities AdminNetworkPolicy AlibabaPlatform AzureWorkloadIdentity BuildCSIVolumes CPMSMachineNamePrefix ConsolePluginContentSecurityPolicy ExternalOIDC ExternalOIDCWithUIDAndExtraClaimMappings GatewayAPI GatewayAPIController HighlyAvailableArbiter ImageVolume IngressControllerLBSubnetsAWS KMSv1 MachineConfigNodes ManagedBootImages ManagedBootImagesAWS MetricsCollectionProfiles NetworkDiagnosticsConfig NetworkLiveMigration NetworkSegmentation PinnedImages ProcMountType RouteAdvertisements RouteExternalCertificate ServiceAccountTokenNodeBinding SetEIPForNLBIngressController SigstoreImageVerification StoragePerformantSecurityPolicy UpgradeStatus UserNamespacesPodSecurityStandards UserNamespacesSupport VSphereMultiDisk VSphereMultiNetworks AWSClusterHostedDNS AWSClusterHostedDNSInstall AWSDedicatedHosts AWSServiceLBNetworkSecurityGroup AutomatedEtcdBackup AzureClusterHostedDNSInstall AzureDedicatedHosts AzureMultiDisk BootImageSkewEnforcement BootcNodeManagement ClusterAPIInstall ClusterAPIInstallIBMCloud ClusterMonitoringConfig ClusterVersionOperatorConfiguration DNSNameResolver DualReplica DyanmicServiceEndpointIBMCloud DynamicResourceAllocation EtcdBackendQuota EventedPLEG Example Example2 ExternalSnapshotMetadata GCPClusterHostedDNS GCPClusterHostedDNSInstall GCPCustomAPIEndpoints GCPCustomAPIEndpointsInstall ImageModeStatusReporting ImageStreamImportMode IngressControllerDynamicConfigurationManager InsightsConfig InsightsConfigAPI InsightsOnDemandDataGather IrreconcilableMachineConfig KMSEncryptionProvider MachineAPIMigration MachineAPIOperatorDisableMachineHealthCheckController ManagedBootImagesAzure ManagedBootImagesvSphere MaxUnavailableStatefulSet MinimumKubeletVersion MixedCPUsAllocation MultiArchInstallAzure MultiDiskSetup MutatingAdmissionPolicy NewOLM NewOLMCatalogdAPIV1Metas NewOLMOwnSingleNamespace NewOLMPreflightPermissionChecks NewOLMWebhookProviderOpenshiftServiceCA NoRegistryClusterOperations NodeSwap NutanixMultiSubnets OVNObservability OpenShiftPodSecurityAdmission PreconfiguredUDNAddresses SELinuxMount ShortCertRotation SignatureStores SigstoreImageVerificationPKI TranslateStreamCloseWebsocketRequests VSphereConfigurableMaxAllowedBlockVolumesPerNode VSphereHostVMGroupZonal VSphereMixedNodeEnv VolumeAttributesClass VolumeGroupSnapshot] I0704 06:58:32.228364 1 requestheader_controller.go:180] Starting RequestHeaderAuthRequestController I0704 06:58:32.228432 1 shared_informer.go:313] Waiting for caches to sync for RequestHeaderAuthRequestController I0704 06:58:32.228618 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::client-ca-file" I0704 06:58:32.228712 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0704 06:58:32.228807 1 secure_serving.go:213] Serving securely on [::]:8443 I0704 06:58:32.229121 1 configmap_cafile_content.go:205] "Starting controller" name="client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file" I0704 06:58:32.229174 1 shared_informer.go:313] Waiting for caches to sync for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file I0704 06:58:32.229566 1 dynamic_serving_content.go:135] "Starting controller" name="serving-cert::/tmp/serving-cert-2425336428/tls.crt::/tmp/serving-cert-2425336428/tls.key" I0704 06:58:32.229565 1 tlsconfig.go:243] "Starting DynamicServingCertificateController" W0704 06:58:32.234251 1 configmapobserver.go:64] Cannot get the configuration config map: configmaps "insights-config" not found. Default configuration is used. I0704 06:58:32.234276 1 secretconfigobserver.go:216] Legacy configuration set: enabled=false endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=false reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0704 06:58:32.234367 1 base_controller.go:76] Waiting for caches to sync for ConfigController I0704 06:58:32.239528 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 06:58:32.239548 1 secretconfigobserver.go:204] Legacy configuration updated: enabled=true endpoint=https://console.redhat.com/api/ingress/v1/upload conditional_gatherer_endpoint=https://console.redhat.com/api/gathering/v2/%s/gathering_rules interval=2h0m0s token=true reportEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports initialPollingDelay=1m0s minRetryTime=30s pollingTimeout=50m0s processingStatusEndpoint=https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/request/%s/status I0704 06:58:32.243907 1 secretconfigobserver.go:119] support secret does not exist I0704 06:58:32.248397 1 secretconfigobserver.go:249] Found cloud.openshift.com token I0704 06:58:32.252824 1 secretconfigobserver.go:119] support secret does not exist I0704 06:58:32.255147 1 recorder.go:161] Pruning old reports every 7h46m9s, max age is 288h0m0s I0704 06:58:32.259737 1 controllerstatus.go:80] name=insightsuploader healthy=true reason= message= I0704 06:58:32.259758 1 insightsuploader.go:86] Reporting status periodically to https://console.redhat.com/api/ingress/v1/upload every 2h0m0s, starting in 1m30s I0704 06:58:32.259771 1 controllerstatus.go:80] name=insightsreport healthy=true reason= message= I0704 06:58:32.259776 1 periodic.go:209] Running clusterconfig gatherer I0704 06:58:32.259786 1 insightsreport.go:296] Starting report retriever I0704 06:58:32.259796 1 insightsreport.go:298] Insights analysis reports will be downloaded from the https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports endpoint with a delay of 1m0s I0704 06:58:32.259828 1 tasks_processing.go:45] number of workers: 32 I0704 06:58:32.259873 1 tasks_processing.go:69] worker 31 listening for tasks. I0704 06:58:32.259882 1 tasks_processing.go:71] worker 31 working on jaegers task. I0704 06:58:32.259885 1 tasks_processing.go:69] worker 0 listening for tasks. I0704 06:58:32.259890 1 tasks_processing.go:69] worker 23 listening for tasks. I0704 06:58:32.259897 1 tasks_processing.go:69] worker 8 listening for tasks. I0704 06:58:32.259899 1 tasks_processing.go:69] worker 7 listening for tasks. I0704 06:58:32.259903 1 tasks_processing.go:69] worker 16 listening for tasks. I0704 06:58:32.259907 1 tasks_processing.go:69] worker 9 listening for tasks. I0704 06:58:32.259911 1 tasks_processing.go:69] worker 17 listening for tasks. I0704 06:58:32.259914 1 tasks_processing.go:69] worker 10 listening for tasks. I0704 06:58:32.259912 1 tasks_processing.go:69] worker 3 listening for tasks. I0704 06:58:32.259923 1 tasks_processing.go:69] worker 18 listening for tasks. I0704 06:58:32.259920 1 tasks_processing.go:69] worker 4 listening for tasks. I0704 06:58:32.259924 1 tasks_processing.go:69] worker 1 listening for tasks. I0704 06:58:32.259932 1 tasks_processing.go:69] worker 2 listening for tasks. I0704 06:58:32.259929 1 tasks_processing.go:69] worker 6 listening for tasks. I0704 06:58:32.259939 1 tasks_processing.go:69] worker 19 listening for tasks. I0704 06:58:32.259942 1 tasks_processing.go:69] worker 5 listening for tasks. I0704 06:58:32.259948 1 tasks_processing.go:69] worker 27 listening for tasks. I0704 06:58:32.259951 1 tasks_processing.go:69] worker 21 listening for tasks. I0704 06:58:32.259951 1 tasks_processing.go:69] worker 22 listening for tasks. I0704 06:58:32.259958 1 tasks_processing.go:69] worker 13 listening for tasks. I0704 06:58:32.259959 1 tasks_processing.go:69] worker 20 listening for tasks. I0704 06:58:32.259962 1 tasks_processing.go:69] worker 25 listening for tasks. I0704 06:58:32.259958 1 tasks_processing.go:69] worker 12 listening for tasks. I0704 06:58:32.259989 1 tasks_processing.go:69] worker 29 listening for tasks. I0704 06:58:32.259990 1 tasks_processing.go:69] worker 26 listening for tasks. I0704 06:58:32.259992 1 tasks_processing.go:69] worker 30 listening for tasks. I0704 06:58:32.259996 1 tasks_processing.go:69] worker 28 listening for tasks. I0704 06:58:32.259959 1 tasks_processing.go:69] worker 24 listening for tasks. I0704 06:58:32.259877 1 tasks_processing.go:69] worker 15 listening for tasks. I0704 06:58:32.259986 1 tasks_processing.go:69] worker 14 listening for tasks. I0704 06:58:32.260012 1 tasks_processing.go:69] worker 11 listening for tasks. I0704 06:58:32.260068 1 tasks_processing.go:71] worker 11 working on machine_sets task. I0704 06:58:32.260074 1 tasks_processing.go:71] worker 6 working on networks task. I0704 06:58:32.260080 1 tasks_processing.go:71] worker 19 working on pod_network_connectivity_checks task. I0704 06:58:32.260087 1 tasks_processing.go:71] worker 17 working on clusterroles task. I0704 06:58:32.260101 1 tasks_processing.go:71] worker 30 working on openshift_machine_api_events task. I0704 06:58:32.260143 1 tasks_processing.go:71] worker 15 working on node_logs task. I0704 06:58:32.260149 1 tasks_processing.go:71] worker 18 working on operators task. I0704 06:58:32.260201 1 tasks_processing.go:71] worker 10 working on container_images task. I0704 06:58:32.260075 1 tasks_processing.go:71] worker 25 working on config_maps task. I0704 06:58:32.260424 1 tasks_processing.go:71] worker 5 working on version task. I0704 06:58:32.260698 1 tasks_processing.go:71] worker 14 working on aggregated_monitoring_cr_names task. I0704 06:58:32.260774 1 tasks_processing.go:71] worker 4 working on nodes task. I0704 06:58:32.260699 1 tasks_processing.go:71] worker 28 working on pdbs task. I0704 06:58:32.261143 1 tasks_processing.go:71] worker 0 working on certificate_signing_requests task. I0704 06:58:32.261271 1 tasks_processing.go:71] worker 1 working on openstack_version task. I0704 06:58:32.261359 1 tasks_processing.go:71] worker 29 working on mutating_webhook_configurations task. I0704 06:58:32.261413 1 tasks_processing.go:71] worker 2 working on overlapping_namespace_uids task. I0704 06:58:32.261283 1 tasks_processing.go:71] worker 3 working on ceph_cluster task. I0704 06:58:32.261863 1 tasks_processing.go:71] worker 27 working on image_pruners task. I0704 06:58:32.262148 1 tasks_processing.go:71] worker 21 working on storage_classes task. I0704 06:58:32.262177 1 tasks_processing.go:71] worker 7 working on machine_autoscalers task. I0704 06:58:32.262217 1 tasks_processing.go:71] worker 26 working on image_registries task. I0704 06:58:32.262285 1 tasks_processing.go:71] worker 16 working on storage_cluster task. I0704 06:58:32.262514 1 tasks_processing.go:71] worker 23 working on cluster_apiserver task. I0704 06:58:32.262537 1 tasks_processing.go:71] worker 9 working on nodenetworkconfigurationpolicies task. I0704 06:58:32.262548 1 tasks_processing.go:71] worker 12 working on support_secret task. I0704 06:58:32.262652 1 tasks_processing.go:71] worker 22 working on machine_configs task. I0704 06:58:32.262711 1 tasks_processing.go:71] worker 8 working on dvo_metrics task. I0704 06:58:32.262821 1 tasks_processing.go:71] worker 13 working on schedulers task. I0704 06:58:32.260499 1 tasks_processing.go:71] worker 24 working on metrics task. W0704 06:58:32.262917 1 gather_most_recent_metrics.go:64] Unable to load metrics client, no metrics will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0704 06:58:32.262937 1 tasks_processing.go:71] worker 24 working on cost_management_metrics_configs task. I0704 06:58:32.263009 1 tasks_processing.go:71] worker 20 working on sap_datahubs task. I0704 06:58:32.263092 1 gather.go:177] gatherer "clusterconfig" function "metrics" took 57.492µs to process 0 records I0704 06:58:32.263602 1 tasks_processing.go:71] worker 31 working on crds task. I0704 06:58:32.263656 1 gather.go:177] gatherer "clusterconfig" function "jaegers" took 3.710892ms to process 0 records E0704 06:58:32.263666 1 gather.go:140] gatherer "clusterconfig" function "pod_network_connectivity_checks" failed with the error: the server could not find the requested resource (get podnetworkconnectivitychecks.controlplane.operator.openshift.io) I0704 06:58:32.263676 1 gather.go:177] gatherer "clusterconfig" function "pod_network_connectivity_checks" took 3.562115ms to process 0 records I0704 06:58:32.263682 1 tasks_processing.go:71] worker 19 working on ingress task. I0704 06:58:32.265673 1 controller.go:128] Initializing last reported time to 0001-01-01T00:00:00Z I0704 06:58:32.265691 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0704 06:58:32.265695 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0704 06:58:32.265698 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0704 06:58:32.265728 1 controller.go:489] The operator is still being initialized I0704 06:58:32.265734 1 controller.go:512] The operator is healthy I0704 06:58:32.266988 1 tasks_processing.go:71] worker 11 working on proxies task. I0704 06:58:32.267023 1 gather.go:177] gatherer "clusterconfig" function "machine_sets" took 6.878505ms to process 0 records I0704 06:58:32.268250 1 tasks_processing.go:71] worker 1 working on sap_pods task. I0704 06:58:32.268261 1 gather.go:177] gatherer "clusterconfig" function "openstack_version" took 6.939879ms to process 0 records I0704 06:58:32.271264 1 tasks_processing.go:71] worker 6 working on service_accounts task. I0704 06:58:32.271470 1 recorder.go:75] Recording config/network with fingerprint=59825101d40177b73658cd7338fe87f7c331ebf0b5e748e3a1f41a2f8f19af49 I0704 06:58:32.271485 1 gather.go:177] gatherer "clusterconfig" function "networks" took 11.173124ms to process 1 records I0704 06:58:32.277802 1 tasks_processing.go:71] worker 7 working on feature_gates task. I0704 06:58:32.277815 1 gather.go:177] gatherer "clusterconfig" function "machine_autoscalers" took 15.60435ms to process 0 records I0704 06:58:32.278922 1 tasks_processing.go:71] worker 9 working on oauths task. I0704 06:58:32.278943 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkconfigurationpolicies" took 16.372052ms to process 0 records I0704 06:58:32.279740 1 cluster_transfer.go:83] checking the availability of cluster transfer. Next check is in 12h0m0s I0704 06:58:32.279738 1 sca.go:136] Pulling SCA certificates from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates. Next check is in 8h0m0s W0704 06:58:32.279932 1 operator.go:288] started I0704 06:58:32.280002 1 base_controller.go:76] Waiting for caches to sync for LoggingSyncer I0704 06:58:32.280649 1 tasks_processing.go:71] worker 24 working on operators_pods_and_events task. I0704 06:58:32.280657 1 gather.go:177] gatherer "clusterconfig" function "cost_management_metrics_configs" took 17.697478ms to process 0 records I0704 06:58:32.281194 1 tasks_processing.go:71] worker 16 working on qemu_kubevirt_launcher_logs task. I0704 06:58:32.281206 1 gather.go:177] gatherer "clusterconfig" function "storage_cluster" took 18.886423ms to process 0 records I0704 06:58:32.282411 1 tasks_processing.go:71] worker 20 working on lokistack task. I0704 06:58:32.282448 1 gather.go:177] gatherer "clusterconfig" function "sap_datahubs" took 19.390648ms to process 0 records I0704 06:58:32.286571 1 tasks_processing.go:71] worker 11 working on container_runtime_configs task. I0704 06:58:32.286659 1 recorder.go:75] Recording config/proxy with fingerprint=e93f048779e61e9961628d797cc80f682e1f940ae345ce27a0f800d57010d439 I0704 06:58:32.286674 1 gather.go:177] gatherer "clusterconfig" function "proxies" took 19.560648ms to process 1 records I0704 06:58:32.286930 1 tasks_processing.go:71] worker 3 working on openstack_controlplanes task. I0704 06:58:32.286957 1 gather.go:177] gatherer "clusterconfig" function "ceph_cluster" took 25.291045ms to process 0 records I0704 06:58:32.287133 1 tasks_processing.go:71] worker 30 working on sap_config task. I0704 06:58:32.287176 1 gather.go:177] gatherer "clusterconfig" function "openshift_machine_api_events" took 27.022758ms to process 0 records I0704 06:58:32.287371 1 tasks_processing.go:71] worker 27 working on silenced_alerts task. W0704 06:58:32.287399 1 gather_silenced_alerts.go:38] Unable to load alerts client, no alerts will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0704 06:58:32.287602 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/imagepruner/cluster with fingerprint=7b19c1f93d5de1d0678c752624802c8ee99d69f85e6de4196b39fdce96687bc6 I0704 06:58:32.287616 1 gather.go:177] gatherer "clusterconfig" function "image_pruners" took 25.48723ms to process 1 records I0704 06:58:32.287626 1 gather.go:177] gatherer "clusterconfig" function "silenced_alerts" took 28.335µs to process 0 records I0704 06:58:32.287638 1 tasks_processing.go:71] worker 27 working on olm_operators task. I0704 06:58:32.293453 1 tasks_processing.go:71] worker 20 working on ingress_certificates task. I0704 06:58:32.293474 1 gather.go:177] gatherer "clusterconfig" function "lokistack" took 11.024928ms to process 0 records I0704 06:58:32.293801 1 tasks_processing.go:71] worker 1 working on openstack_dataplanedeployments task. I0704 06:58:32.293816 1 gather.go:177] gatherer "clusterconfig" function "sap_pods" took 25.537585ms to process 0 records I0704 06:58:32.295419 1 tasks_processing.go:71] worker 12 working on image task. E0704 06:58:32.295434 1 gather.go:140] gatherer "clusterconfig" function "support_secret" failed with the error: secrets "support" not found I0704 06:58:32.295447 1 gather.go:177] gatherer "clusterconfig" function "support_secret" took 32.851377ms to process 0 records I0704 06:58:32.295766 1 tasks_processing.go:71] worker 23 working on machine_config_pools task. I0704 06:58:32.295984 1 recorder.go:75] Recording config/apiserver with fingerprint=2836c80c69c64e0dd509c84d9e2e6c91def1a53cef42c0f1431f529a0aba2793 I0704 06:58:32.296005 1 gather.go:177] gatherer "clusterconfig" function "cluster_apiserver" took 33.234844ms to process 1 records I0704 06:58:32.296693 1 tasks_processing.go:71] worker 26 working on machines task. I0704 06:58:32.297037 1 recorder.go:75] Recording config/clusteroperator/imageregistry.operator.openshift.io/config/cluster with fingerprint=2ef122c4b64f0892c0fb310868d2fd22248f9bc9a1919bf9eb1d9db73b058e31 I0704 06:58:32.297056 1 gather.go:177] gatherer "clusterconfig" function "image_registries" took 34.461174ms to process 1 records I0704 06:58:32.300550 1 tasks_processing.go:71] worker 1 working on nodenetworkstates task. I0704 06:58:32.300559 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanedeployments" took 6.729697ms to process 0 records I0704 06:58:32.307782 1 tasks_processing.go:71] worker 26 working on validating_webhook_configurations task. E0704 06:58:32.307798 1 gather.go:140] gatherer "clusterconfig" function "machines" failed with the error: machines.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machines" in API group "machine.openshift.io" at the cluster scope I0704 06:58:32.307815 1 gather.go:177] gatherer "clusterconfig" function "machines" took 11.071393ms to process 0 records I0704 06:58:32.316723 1 tasks_processing.go:71] worker 13 working on number_of_pods_and_netnamespaces_with_sdn_annotations task. I0704 06:58:32.316748 1 prometheus_rules.go:88] Prometheus rules successfully created I0704 06:58:32.316852 1 recorder.go:75] Recording config/schedulers/cluster with fingerprint=1162fd56c36fb9e7e44eb22ea52865771d89a7cea0935ee153b52e9a8fade0ad I0704 06:58:32.316880 1 gather.go:177] gatherer "clusterconfig" function "schedulers" took 53.890409ms to process 1 records I0704 06:58:32.316990 1 tasks_processing.go:71] worker 28 working on active_alerts task. I0704 06:58:32.317015 1 recorder.go:75] Recording config/pdbs/openshift-image-registry/image-registry with fingerprint=dd9906a552564f7eecc1161f09cb3c7cb7ad3b5605eb00929a239677c5ae7064 W0704 06:58:32.317024 1 gather_active_alerts.go:54] Unable to load alerts client, no alerts will be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0704 06:58:32.317037 1 recorder.go:75] Recording config/pdbs/openshift-ingress/router-default with fingerprint=70daacc8661dc26941588522fedc7171390991c9198cb7620be4ef2bf39fa501 I0704 06:58:32.317054 1 recorder.go:75] Recording config/pdbs/openshift-operator-lifecycle-manager/packageserver-pdb with fingerprint=29f5408b79970e34eabf3e1300bcefc9ff38af1a603a8c7ab69648bfa760b4eb I0704 06:58:32.317064 1 gather.go:177] gatherer "clusterconfig" function "pdbs" took 55.792554ms to process 3 records I0704 06:58:32.317074 1 gather.go:177] gatherer "clusterconfig" function "active_alerts" took 28.283µs to process 0 records I0704 06:58:32.317080 1 tasks_processing.go:71] worker 28 working on openshift_logging task. I0704 06:58:32.317586 1 tasks_processing.go:71] worker 1 working on machine_healthchecks task. I0704 06:58:32.317617 1 gather.go:177] gatherer "clusterconfig" function "nodenetworkstates" took 17.023104ms to process 0 records I0704 06:58:32.317767 1 controller.go:212] Source periodic-conditional *controllerstatus.Simple is not ready I0704 06:58:32.317813 1 controller.go:212] Source periodic-workloads *controllerstatus.Simple is not ready I0704 06:58:32.317817 1 controller.go:212] Source scaController *sca.Controller is not ready I0704 06:58:32.317821 1 controller.go:212] Source clusterTransferController *clustertransfer.Controller is not ready I0704 06:58:32.317823 1 controller.go:212] Source periodic-clusterconfig *controllerstatus.Simple is not ready I0704 06:58:32.317841 1 controller.go:489] The operator is still being initialized I0704 06:58:32.317849 1 controller.go:512] The operator is healthy I0704 06:58:32.317990 1 tasks_processing.go:71] worker 30 working on monitoring_persistent_volumes task. I0704 06:58:32.318007 1 gather.go:177] gatherer "clusterconfig" function "sap_config" took 30.839932ms to process 0 records I0704 06:58:32.318255 1 gather.go:177] gatherer "clusterconfig" function "openstack_controlplanes" took 31.057906ms to process 0 records I0704 06:58:32.318272 1 gather.go:177] gatherer "clusterconfig" function "container_runtime_configs" took 31.431715ms to process 0 records I0704 06:58:32.318279 1 gather.go:177] gatherer "clusterconfig" function "node_logs" took 57.882401ms to process 0 records I0704 06:58:32.318316 1 tasks_processing.go:71] worker 11 working on openstack_dataplanenodesets task. I0704 06:58:32.318480 1 tasks_processing.go:71] worker 15 working on infrastructures task. I0704 06:58:32.318569 1 tasks_processing.go:71] worker 9 working on tsdb_status task. W0704 06:58:32.318637 1 gather_prometheus_tsdb_status.go:38] Unable to load metrics client, tsdb status cannot be collected: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0704 06:58:32.318503 1 tasks_processing.go:71] worker 3 working on authentication task. I0704 06:58:32.318828 1 recorder.go:75] Recording config/oauth with fingerprint=8029b2ec49267b2a44188aa4223d8ac62dbd2b6aae1ac8f9e092b505ac9bc494 I0704 06:58:32.318887 1 gather.go:177] gatherer "clusterconfig" function "oauths" took 39.20224ms to process 1 records I0704 06:58:32.319038 1 recorder.go:75] Recording config/image with fingerprint=20ea9f3317a01da376395e77d45d37142965a57cec64ccea95e21222fb08ce0c I0704 06:58:32.319055 1 gather.go:177] gatherer "clusterconfig" function "image" took 22.877779ms to process 1 records I0704 06:58:32.319133 1 tasks_processing.go:71] worker 12 working on install_plans task. I0704 06:58:32.319274 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/admin with fingerprint=30b0b891c1c4014959bf8f68ae0b99ca16e650d71f1df224727034e617206db2 I0704 06:58:32.319418 1 recorder.go:75] Recording cluster-scoped-resources/rbac.authorization.k8s.io/clusterroles/edit with fingerprint=f4f8777eaadb155574d32ffeb4a98bbff69d713f2455ead89aeee858d8d79caf I0704 06:58:32.319446 1 gather.go:177] gatherer "clusterconfig" function "clusterroles" took 58.310536ms to process 2 records I0704 06:58:32.319471 1 gather.go:177] gatherer "clusterconfig" function "tsdb_status" took 50.668µs to process 0 records I0704 06:58:32.319507 1 tasks_processing.go:74] worker 7 stopped. I0704 06:58:32.319285 1 tasks_processing.go:74] worker 17 stopped. I0704 06:58:32.319561 1 tasks_processing.go:74] worker 9 stopped. I0704 06:58:32.321766 1 recorder.go:75] Recording config/featuregate with fingerprint=cf47dc747ea1d318baf869aa39171396b7950ee2f7e9da6f6f56702f9e2aa72b I0704 06:58:32.322676 1 gather.go:177] gatherer "clusterconfig" function "feature_gates" took 41.080814ms to process 1 records I0704 06:58:32.322776 1 tasks_processing.go:74] worker 29 stopped. I0704 06:58:32.323347 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/aws-pod-identity with fingerprint=69049bc00a24fbd0f362e3cd09eea5fa58d9372c3d372644f5b032cc48570521 I0704 06:58:32.323408 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-podimagespec-mutation with fingerprint=1983ba7fea041db62658b13cc50f1744f115f8dba3644501a25fcf2bac28d12c I0704 06:58:32.323454 1 recorder.go:75] Recording config/mutatingwebhookconfigurations/sre-service-mutation with fingerprint=2b8f0ebb4822df2216298e7a791a48f47dae8821e7e10645c2b9a789f328a1cc I0704 06:58:32.323468 1 gather.go:177] gatherer "clusterconfig" function "mutating_webhook_configurations" took 57.55472ms to process 3 records I0704 06:58:32.323492 1 recorder.go:75] Recording config/namespaces_with_overlapping_uids with fingerprint=4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945 I0704 06:58:32.323505 1 gather.go:177] gatherer "clusterconfig" function "overlapping_namespace_uids" took 57.636887ms to process 1 records I0704 06:58:32.323562 1 tasks_processing.go:74] worker 2 stopped. I0704 06:58:32.323659 1 recorder.go:75] Recording config/ingress with fingerprint=1abd74d5add2987d5a11c72c06dce8c57ef75f27f1f250e677cff09379411010 I0704 06:58:32.323675 1 gather.go:177] gatherer "clusterconfig" function "ingress" took 55.588716ms to process 1 records I0704 06:58:32.323683 1 gather.go:177] gatherer "clusterconfig" function "certificate_signing_requests" took 60.408395ms to process 0 records I0704 06:58:32.323691 1 tasks_processing.go:74] worker 0 stopped. I0704 06:58:32.323697 1 tasks_processing.go:74] worker 19 stopped. I0704 06:58:32.323719 1 tasks_processing.go:74] worker 21 stopped. I0704 06:58:32.323819 1 recorder.go:75] Recording config/storage/storageclasses/gp2-csi with fingerprint=d6fb25a36003c7adde50e8f822e37a39a8da1b276dc861d289f83e485a98b514 I0704 06:58:32.323847 1 recorder.go:75] Recording config/storage/storageclasses/gp3-csi with fingerprint=c30799fb377ffcf17c8a13d85e518d56c796623ad4dfd6ea378dd2a06084303e I0704 06:58:32.323858 1 gather.go:177] gatherer "clusterconfig" function "storage_classes" took 61.560233ms to process 2 records E0704 06:58:32.323871 1 gather.go:140] gatherer "clusterconfig" function "machine_healthchecks" failed with the error: machinehealthchecks.machine.openshift.io is forbidden: User "system:serviceaccount:openshift-insights:gather" cannot list resource "machinehealthchecks" in API group "machine.openshift.io" at the cluster scope I0704 06:58:32.323882 1 gather.go:177] gatherer "clusterconfig" function "machine_healthchecks" took 6.091242ms to process 0 records I0704 06:58:32.323889 1 gather.go:177] gatherer "clusterconfig" function "openshift_logging" took 6.70612ms to process 0 records I0704 06:58:32.323899 1 tasks_processing.go:74] worker 28 stopped. I0704 06:58:32.323904 1 tasks_processing.go:74] worker 1 stopped. W0704 06:58:32.325362 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0704 06:58:32.327777 1 tasks_processing.go:74] worker 11 stopped. I0704 06:58:32.327794 1 gather.go:177] gatherer "clusterconfig" function "openstack_dataplanenodesets" took 9.437273ms to process 0 records I0704 06:58:32.328596 1 shared_informer.go:320] Caches are synced for RequestHeaderAuthRequestController I0704 06:58:32.329243 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::client-ca-file I0704 06:58:32.329262 1 shared_informer.go:320] Caches are synced for client-ca::kube-system::extension-apiserver-authentication::requestheader-client-ca-file E0704 06:58:32.329710 1 cluster_transfer.go:95] failed to pull cluster transfer: unable to retrieve cluster transfer data from https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/: Get "https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/?search=cluster_uuid+is+%270c903966-ad9e-4231-b635-9b7f8d3b3c8f%27+and+status+is+%27accepted%27": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.14:37315->172.30.0.10:53: read: connection refused I0704 06:58:32.329727 1 controllerstatus.go:80] name=clusterTransferController healthy=true reason=Disconnected message=failed to pull cluster transfer: unable to retrieve cluster transfer data from https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/: Get "https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/?search=cluster_uuid+is+%270c903966-ad9e-4231-b635-9b7f8d3b3c8f%27+and+status+is+%27accepted%27": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.14:37315->172.30.0.10:53: read: connection refused I0704 06:58:32.329760 1 tasks_processing.go:74] worker 26 stopped. I0704 06:58:32.329935 1 recorder.go:75] Recording config/validatingwebhookconfigurations/multus.openshift.io with fingerprint=88e0fb0c67a116e75885000cc5803e7c5d423bbdf06a3a4f0b17ee3182c5618f I0704 06:58:32.330061 1 recorder.go:75] Recording config/validatingwebhookconfigurations/network-node-identity.openshift.io with fingerprint=e9abc5d33e1e2aede30507f68dba5a4bf0737a7cc3321dc26d7eceed58a8ed54 I0704 06:58:32.330084 1 recorder.go:75] Recording config/validatingwebhookconfigurations/performance-addon-operator with fingerprint=a8d9a8c07744503ed5cc1b1430eed6623004c69c5f63da974efb8e8329113ac8 I0704 06:58:32.330110 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterrolebindings-validation with fingerprint=0f45db2c39afe6ae1f0a73566edd53c4f4a975f5aa435d4e45a7493afb44adf8 I0704 06:58:32.330133 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-clusterroles-validation with fingerprint=40008121386e9f53a359377a139ed4a2f9ad5a93a82432aa500a0d2cb5788313 I0704 06:58:32.330156 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-ingress-config-validation with fingerprint=8100969ab939ea514b8c48bbadfd496a5db90a57ba4080242e317f08d4c0efe0 I0704 06:58:32.330192 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-network-operator-validation with fingerprint=a19caa5947085dce02dcdf4dcf729be2e1ae27a88fea5e21073bd912b48231a5 I0704 06:58:32.330225 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-regular-user-validation with fingerprint=089891f5a78d4574d9e9cd1a9ea92049617fc7a860d33e95471df69296b6b249 I0704 06:58:32.330258 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-scc-validation with fingerprint=3ae27795d79e58da1d3c411cbbe88c80a772b81e98385247079cb04cc3929c8d I0704 06:58:32.330282 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-serviceaccount-validation with fingerprint=36985ff54900d8789b6a26eef41e54d428405af0bb75e52d86cbbfba47caa261 I0704 06:58:32.330305 1 recorder.go:75] Recording config/validatingwebhookconfigurations/sre-techpreviewnoupgrade-validation with fingerprint=48b2d71dbba520283d119020d94e8698b8bd57bbf9bb1dd51facb00cdb478c60 I0704 06:58:32.330313 1 gather.go:177] gatherer "clusterconfig" function "validating_webhook_configurations" took 21.95322ms to process 11 records I0704 06:58:32.331878 1 tasks_processing.go:74] worker 30 stopped. I0704 06:58:32.331952 1 gather.go:177] gatherer "clusterconfig" function "monitoring_persistent_volumes" took 13.819146ms to process 0 records I0704 06:58:32.332740 1 tasks_processing.go:74] worker 10 stopped. I0704 06:58:32.334589 1 base_controller.go:82] Caches are synced for ConfigController I0704 06:58:32.334605 1 base_controller.go:119] Starting #1 worker of ConfigController controller ... I0704 06:58:32.335065 1 recorder.go:75] Recording config/pod/openshift-console-operator/console-operator-575cd97545-6bwbj with fingerprint=442f5953c3ee9195ce57c99fdfb521a6254ebf64263fc7cc58135e44b3cb5226 I0704 06:58:32.335332 1 recorder.go:75] Recording config/pod/openshift-ovn-kubernetes/ovnkube-node-9glb6 with fingerprint=9198490357fcff2fa6a9d4207b54ed3d27d61929710ea262b8907644a1b48f92 I0704 06:58:32.335377 1 recorder.go:75] Recording config/running_containers with fingerprint=f2b32ba0879a32c573c8eb516bf99b18fdb34c44f64d194c87444894ae2a76c4 I0704 06:58:32.335386 1 gather.go:177] gatherer "clusterconfig" function "container_images" took 72.484248ms to process 3 records I0704 06:58:32.335469 1 tasks_processing.go:74] worker 15 stopped. I0704 06:58:32.335877 1 recorder.go:75] Recording config/infrastructure with fingerprint=c8977be66261940838196e8b2b30d85c3c3d9bf53b51713a6114836138b29b40 I0704 06:58:32.335888 1 gather.go:177] gatherer "clusterconfig" function "infrastructures" took 15.516395ms to process 1 records I0704 06:58:32.348117 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 06:58:32.348666 1 tasks_processing.go:74] worker 3 stopped. I0704 06:58:32.348894 1 recorder.go:75] Recording config/authentication with fingerprint=efcf2e4c98ea98ae06a6050e973b063fd83b18e2cc72aecb5839613861a657c5 I0704 06:58:32.348909 1 gather.go:177] gatherer "clusterconfig" function "authentication" took 29.967177ms to process 1 records I0704 06:58:32.349061 1 tasks_processing.go:74] worker 31 stopped. I0704 06:58:32.349575 1 recorder.go:75] Recording config/crd/volumesnapshots.snapshot.storage.k8s.io with fingerprint=e78b450a37693cfff524eec9276ffc21f82e902448a87c3d82ee565db52ad85f I0704 06:58:32.349798 1 recorder.go:75] Recording config/crd/volumesnapshotcontents.snapshot.storage.k8s.io with fingerprint=313001f4797487e8c93e1afd22ab4384e5ebd4a8cda33737fed732093666a521 I0704 06:58:32.349809 1 gather.go:177] gatherer "clusterconfig" function "crds" took 85.43495ms to process 2 records I0704 06:58:32.362506 1 tasks_processing.go:74] worker 5 stopped. I0704 06:58:32.362515 1 gather_logs.go:145] no pods in namespace were found I0704 06:58:32.362875 1 recorder.go:75] Recording config/version with fingerprint=9fd09d36c3b5857a3ae2354b702c3e624d39903de15618cbcb518eb32cc363e0 I0704 06:58:32.362892 1 recorder.go:75] Recording config/id with fingerprint=d90f8018bff69f00f643b06a5d65fe36d283a0383c8014320a511c3d52aa4a4a I0704 06:58:32.362899 1 gather.go:177] gatherer "clusterconfig" function "version" took 102.061042ms to process 2 records I0704 06:58:32.362911 1 gather.go:177] gatherer "clusterconfig" function "qemu_kubevirt_launcher_logs" took 81.324327ms to process 0 records I0704 06:58:32.362921 1 tasks_processing.go:74] worker 16 stopped. I0704 06:58:32.377201 1 tasks_processing.go:74] worker 25 stopped. E0704 06:58:32.377220 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "cluster-monitoring-config" not found E0704 06:58:32.377228 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "gateway-mode-config" not found E0704 06:58:32.377234 1 gather.go:140] gatherer "clusterconfig" function "config_maps" failed with the error: configmaps "insights-config" not found I0704 06:58:32.377246 1 recorder.go:75] Recording config/configmaps/openshift-config/installer-images/images.json with fingerprint=26b6661162b099a0f5a279859b4f46c867929a79d9a4a41fde4be4e6fe138018 I0704 06:58:32.377279 1 recorder.go:75] Recording config/configmaps/openshift-config/kube-root-ca.crt/ca.crt with fingerprint=f73fcd48d6f9ebbc83253a1b5f23ef59d0683465ec46e474389103c717467dfb I0704 06:58:32.377283 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/invoker with fingerprint=76b482f683cd3ef9da02debac5b26080a5aeb06ff768ee5c21117514dff29d8a I0704 06:58:32.377287 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-install/version with fingerprint=c93090eb0d2a4736885abeb79c91680cfd01fda46464f83456b085d4dc8239f0 I0704 06:58:32.377291 1 recorder.go:75] Recording config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 I0704 06:58:32.377335 1 recorder.go:75] Recording config/configmaps/openshift-config/rosa-brand-logo/rosa-brand-logo.svg with fingerprint=6ed8ca4dd7a8eee7249182bc006e9649ce84d76c551ddfaaa33e55d8c4cc1ed0 I0704 06:58:32.377345 1 recorder.go:75] Recording config/configmaps/kube-system/cluster-config-v1/install-config with fingerprint=ab3811c6b83fd7b8e920094cfa3080d1b4ee3c35ec4c8379437b21d27bd6608d I0704 06:58:32.377350 1 gather.go:177] gatherer "clusterconfig" function "config_maps" took 116.900072ms to process 7 records I0704 06:58:32.380218 1 base_controller.go:82] Caches are synced for LoggingSyncer I0704 06:58:32.380242 1 base_controller.go:119] Starting #1 worker of LoggingSyncer controller ... I0704 06:58:32.385282 1 tasks_processing.go:74] worker 13 stopped. I0704 06:58:32.385300 1 gather.go:177] gatherer "clusterconfig" function "number_of_pods_and_netnamespaces_with_sdn_annotations" took 68.534325ms to process 0 records I0704 06:58:32.392674 1 tasks_processing.go:74] worker 4 stopped. I0704 06:58:32.393149 1 recorder.go:75] Recording config/node/ip-10-0-0-38.ec2.internal with fingerprint=bddd47b0e2b861d378132de1ebfe3af563a153bc9f9409b77c6e6143a6ada796 I0704 06:58:32.393241 1 recorder.go:75] Recording config/node/ip-10-0-1-129.ec2.internal with fingerprint=3f4268749520a01bd4e92b6e386257e41075fbb5726c92d26f7603d4a3e7ea3d I0704 06:58:32.393321 1 recorder.go:75] Recording config/node/ip-10-0-2-162.ec2.internal with fingerprint=3cae0f8e69eb3da07d4cc1934381f5b1525afdc50f8024bdc755b0bd49af7def I0704 06:58:32.393334 1 gather.go:177] gatherer "clusterconfig" function "nodes" took 131.837707ms to process 3 records I0704 06:58:32.395857 1 tasks_processing.go:74] worker 23 stopped. I0704 06:58:32.395874 1 gather.go:177] gatherer "clusterconfig" function "machine_config_pools" took 100.059478ms to process 0 records I0704 06:58:32.403126 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found I0704 06:58:32.406189 1 requests.go:205] Asking for SCA certificate with "{"arch": ["x86_64"]}" payload I0704 06:58:32.407989 1 gather_cluster_operators.go:184] Unable to get dnsrecords.ingress.operator.openshift.io resource due to: dnsrecords.ingress.operator.openshift.io "default" not found W0704 06:58:32.409966 1 sca.go:161] Failed to pull SCA certs from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: unable to retrieve SCA certs data from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: Post "https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.14:42383->172.30.0.10:53: read: connection refused I0704 06:58:32.409996 1 controllerstatus.go:80] name=scaController healthy=true reason=NonHTTPError message=Failed to pull SCA certs from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: unable to retrieve SCA certs data from https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates: Post "https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates": dial tcp: lookup api.openshift.com on 172.30.0.10:53: read udp 10.128.0.14:42383->172.30.0.10:53: read: connection refused I0704 06:58:32.420096 1 tasks_processing.go:74] worker 27 stopped. I0704 06:58:32.420110 1 gather.go:177] gatherer "clusterconfig" function "olm_operators" took 132.440425ms to process 0 records I0704 06:58:32.449610 1 tasks_processing.go:74] worker 20 stopped. E0704 06:58:32.449634 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret 'router-certs-default' in namespace 'openshift-ingress': secrets "router-certs-default" not found E0704 06:58:32.449643 1 gather.go:140] gatherer "clusterconfig" function "ingress_certificates" failed with the error: failed to get secret '2rb5pqcicn3f4i7uuol1uq6d13p17o2f-primary-cert-bundle-secret' in namespace 'openshift-ingress-operator': secrets "2rb5pqcicn3f4i7uuol1uq6d13p17o2f-primary-cert-bundle-secret" not found I0704 06:58:32.449725 1 recorder.go:75] Recording aggregated/ingress_controllers_certs with fingerprint=77cdbc220f4f255d012defca47b6accd023759b02ad712ba39135b91d27b1aa5 I0704 06:58:32.449739 1 gather.go:177] gatherer "clusterconfig" function "ingress_certificates" took 156.132888ms to process 1 records I0704 06:58:32.455073 1 tasks_processing.go:74] worker 14 stopped. I0704 06:58:32.455090 1 gather.go:177] gatherer "clusterconfig" function "aggregated_monitoring_cr_names" took 194.315673ms to process 0 records I0704 06:58:32.492272 1 tasks_processing.go:74] worker 22 stopped. I0704 06:58:32.492314 1 recorder.go:75] Recording aggregated/unused_machine_configs_count with fingerprint=4bfc9fa984e5dfcd45848faaf05269de7619bf42edf9f781751af5ee05c1a499 I0704 06:58:32.492328 1 gather.go:177] gatherer "clusterconfig" function "machine_configs" took 229.601654ms to process 1 records W0704 06:58:33.322690 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0704 06:58:33.589540 1 gather_cluster_operator_pods_and_events.go:121] Found 18 pods with 21 containers I0704 06:58:33.589558 1 gather_cluster_operator_pods_and_events.go:235] Maximum buffer size: 1198372 bytes I0704 06:58:33.590308 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns container dns-default-g7m68 pod in namespace openshift-dns (previous: false). I0704 06:58:33.713289 1 gather_cluster_operators.go:184] Unable to get configs.samples.operator.openshift.io resource due to: configs.samples.operator.openshift.io "cluster" not found I0704 06:58:33.820190 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-g7m68 pod in namespace openshift-dns for failing operator dns (previous: false): "container \"dns\" in pod \"dns-default-g7m68\" is waiting to start: ContainerCreating" I0704 06:58:33.820210 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"dns\" in pod \"dns-default-g7m68\" is waiting to start: ContainerCreating" I0704 06:58:33.820220 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-rbac-proxy container dns-default-g7m68 pod in namespace openshift-dns (previous: false). I0704 06:58:33.994477 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-g7m68 pod in namespace openshift-dns for failing operator kube-rbac-proxy (previous: false): "container \"kube-rbac-proxy\" in pod \"dns-default-g7m68\" is waiting to start: ContainerCreating" I0704 06:58:33.994495 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"kube-rbac-proxy\" in pod \"dns-default-g7m68\" is waiting to start: ContainerCreating" I0704 06:58:33.994507 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns container dns-default-pcr9d pod in namespace openshift-dns (previous: false). I0704 06:58:34.214761 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-pcr9d pod in namespace openshift-dns for failing operator dns (previous: false): "container \"dns\" in pod \"dns-default-pcr9d\" is waiting to start: ContainerCreating" I0704 06:58:34.214781 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"dns\" in pod \"dns-default-pcr9d\" is waiting to start: ContainerCreating" I0704 06:58:34.214792 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-rbac-proxy container dns-default-pcr9d pod in namespace openshift-dns (previous: false). W0704 06:58:34.323385 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0704 06:58:34.327012 1 tasks_processing.go:74] worker 18 stopped. I0704 06:58:34.327078 1 recorder.go:75] Recording config/clusteroperator/console with fingerprint=2cee08bfe3321cdfb92df7b6a7874833381638fd3bf433fd8f421c492ef20fb3 I0704 06:58:34.327130 1 recorder.go:75] Recording config/clusteroperator/csi-snapshot-controller with fingerprint=c1c95f9ff71708be18765f50ec396e6b4f15ac14cc01d47e5a4f61413950cae2 I0704 06:58:34.327231 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/csisnapshotcontroller/cluster with fingerprint=5adc514f4b63e2f1ecc68bf6f9c0af70c5eea04522a49524e102721b1c41f80e I0704 06:58:34.327283 1 recorder.go:75] Recording config/clusteroperator/dns with fingerprint=f4bd95244d57bdcd1d58a26e8d1fee6707916bea8a1f5b09ed2ac11b9f0a52e1 I0704 06:58:34.327332 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/dns/default with fingerprint=9e7b4ce029030d3d8c3b49af92c556acdcc415000b40d3f969dbdc42c432b47f I0704 06:58:34.327371 1 recorder.go:75] Recording config/clusteroperator/image-registry with fingerprint=53522f91f38a0fe2ee6dd0e5bfe556ca4e4ba3a7718836e0f4917e255d649ee2 I0704 06:58:34.327421 1 recorder.go:75] Recording config/clusteroperator/ingress with fingerprint=eb13c7eda38adbf2a19847f70ffa65e12dfbdb449df47399e208ae8382b2cb71 I0704 06:58:34.327496 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/ingresscontroller/openshift-ingress-operator/default with fingerprint=403997fcb9c85d9ca2eb55bef0f53c66cb0d4dd9889e07934c84bd673b0924b2 I0704 06:58:34.327523 1 recorder.go:75] Recording config/clusteroperator/insights with fingerprint=5a2061e4c75d8b7281ccc6a671f71b2265b7ad3ce7a88641cfc7d66aa89c1e4c I0704 06:58:34.327571 1 recorder.go:75] Recording config/clusteroperator/kube-apiserver with fingerprint=9e7f02350c71020138b7816ca09463c99541dde7a909fd80f4e8d5502c699b22 I0704 06:58:34.327585 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubeapiserver/cluster with fingerprint=51503bf0b784fcf65ea46bcaf1f72ac1a5c4d5dc211934f18f27871efed05762 I0704 06:58:34.327614 1 recorder.go:75] Recording config/clusteroperator/kube-controller-manager with fingerprint=7de7148411583986d1b9e657471ceb85d92243ae52ebbf5792cdcf97f067be94 I0704 06:58:34.327631 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubecontrollermanager/cluster with fingerprint=ce90c0d4f367d7da085074268031798382ae7c54fdcb0a21f15a4818fe308c11 I0704 06:58:34.327657 1 recorder.go:75] Recording config/clusteroperator/kube-scheduler with fingerprint=e7dee1a9be0fb2bd0bcab1baa8c976371de40786cb56eebebe0be74a6473f06e I0704 06:58:34.327670 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubescheduler/cluster with fingerprint=f2940fb9fd20c19951dfc295eb363b7fba0c505f5ae61f01967a063099e6b60a I0704 06:58:34.327702 1 recorder.go:75] Recording config/clusteroperator/kube-storage-version-migrator with fingerprint=10f3bd181a44eabfe7527b3050ff54e90555b8068df2a7f556e59153ba4ce8f7 I0704 06:58:34.327715 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/kubestorageversionmigrator/cluster with fingerprint=9351181aa7e6ada41ef581ab31e13516c6b934cc95710154bafb2eb222cb58db I0704 06:58:34.327737 1 recorder.go:75] Recording config/clusteroperator/monitoring with fingerprint=3a6f758d2f347875c9355f11d5c174c1e2626d7395f8a4c2b067f0bfc9f48a4d I0704 06:58:34.327937 1 recorder.go:75] Recording config/clusteroperator/network with fingerprint=0bc62bf11432e621495a00c4b4c91fc27d9d990647753fec1f99c7bebd984eac I0704 06:58:34.327954 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/ovn with fingerprint=626a89d20e0deaed5b6dfb533acfe65f4bb1618bd200a703b62e60c5d16d94ab I0704 06:58:34.327983 1 recorder.go:75] Recording config/clusteroperator/network.operator.openshift.io/operatorpki/openshift-ovn-kubernetes/signer with fingerprint=90410b16914712b85b3c4578716ad8c0ae072e688f4cd1e022bf76f20da3506d I0704 06:58:34.328066 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/network/cluster with fingerprint=a93d15eaecb455a0e40ecb2826eeecc1533899204ddd3c3921d15ab70af7ae75 I0704 06:58:34.328114 1 recorder.go:75] Recording config/clusteroperator/node-tuning with fingerprint=8f8fcd554b96ea6389520bfe5babdc454f95f3a0ea5077106f8c2ba0b2feb7f3 I0704 06:58:34.328157 1 recorder.go:75] Recording config/clusteroperator/openshift-apiserver with fingerprint=221c40f5882d137a0ff796f6c5881ec1a97e8550ddcfd6339435cff5e64c3ca1 I0704 06:58:34.328173 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftapiserver/cluster with fingerprint=e712e6cf27339b441e4ed1f4cde91dbde7e952698ba93407e4457db63a4a4c76 I0704 06:58:34.328199 1 recorder.go:75] Recording config/clusteroperator/openshift-controller-manager with fingerprint=2bc8074a2e70714dcf2ab5d02fa64ecc0b91af7bc81feaf4388db83efe04f173 I0704 06:58:34.328215 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/openshiftcontrollermanager/cluster with fingerprint=d71a0f4672f9b45d9fc8293bf1687afc650fd28d32e2e30de27523fe7b4eadf7 I0704 06:58:34.328240 1 recorder.go:75] Recording config/clusteroperator/openshift-samples with fingerprint=430dfb290134588d2bcf8f53a2d92a8790b1398b694bc91e3f54114d023c1da1 I0704 06:58:34.328264 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager with fingerprint=919a1dbb8a53716664cdd41ffefd114eb170eb2bc9677851a1eb832ffe63ed8b I0704 06:58:34.328293 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-catalog with fingerprint=9f10cfcf4f66d441788dbf766ddcc001554f4d46e06c1593262316982f1ba9d7 I0704 06:58:34.328319 1 recorder.go:75] Recording config/clusteroperator/operator-lifecycle-manager-packageserver with fingerprint=14dab72b1efd70522193722abbb6287b7ebd4d6dc790cff94ffe08117f8cd790 I0704 06:58:34.328357 1 recorder.go:75] Recording config/clusteroperator/service-ca with fingerprint=90098eac26329db4a9c2dd84432ddd173c6d3fdf482ef471b836285b876ccf6f I0704 06:58:34.328374 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/serviceca/cluster with fingerprint=812f7edc2cdb30e61e7f2b29454357a40b1a507a4b0c2b7729193b67f0e3b4aa I0704 06:58:34.328427 1 recorder.go:75] Recording config/clusteroperator/storage with fingerprint=9019a6d78dbff47c1e68b38704c674bb6785e8825021ee042814ddadc4c6fe27 I0704 06:58:34.328455 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/clustercsidriver/ebs.csi.aws.com with fingerprint=510064d6f6bcced87ab5bd2ddaff3d0edd7f93f4a4f7af2641f29fc53ffab21e I0704 06:58:34.328472 1 recorder.go:75] Recording config/clusteroperator/operator.openshift.io/storage/cluster with fingerprint=8e480f8c1ce1b39baac42d8ec780c57c2592929ae0c801b61ffad49ba13f33ad I0704 06:58:34.328483 1 gather.go:177] gatherer "clusterconfig" function "operators" took 2.066841169s to process 36 records I0704 06:58:34.394667 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for dns-default-pcr9d pod in namespace openshift-dns for failing operator kube-rbac-proxy (previous: false): "container \"kube-rbac-proxy\" in pod \"dns-default-pcr9d\" is waiting to start: ContainerCreating" I0704 06:58:34.394686 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"kube-rbac-proxy\" in pod \"dns-default-pcr9d\" is waiting to start: ContainerCreating" I0704 06:58:34.394697 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-r2bhj pod in namespace openshift-dns (previous: false). I0704 06:58:34.595267 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0704 06:58:34.595288 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-s5jxf pod in namespace openshift-dns (previous: false). I0704 06:58:34.795246 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0704 06:58:34.795266 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for dns-node-resolver container node-resolver-v8n2n pod in namespace openshift-dns (previous: false). I0704 06:58:35.030610 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0704 06:58:35.030633 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-597d5ff6c5-9h6n5 pod in namespace openshift-image-registry (previous: false). I0704 06:58:35.201123 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for image-registry-597d5ff6c5-9h6n5 pod in namespace openshift-image-registry for failing operator registry (previous: false): "container \"registry\" in pod \"image-registry-597d5ff6c5-9h6n5\" is waiting to start: ContainerCreating" I0704 06:58:35.201145 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"registry\" in pod \"image-registry-597d5ff6c5-9h6n5\" is waiting to start: ContainerCreating" I0704 06:58:35.201158 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-597d5ff6c5-zq2s2 pod in namespace openshift-image-registry (previous: false). W0704 06:58:35.322678 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0704 06:58:35.402762 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for image-registry-597d5ff6c5-zq2s2 pod in namespace openshift-image-registry for failing operator registry (previous: false): "container \"registry\" in pod \"image-registry-597d5ff6c5-zq2s2\" is waiting to start: ContainerCreating" I0704 06:58:35.402858 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"registry\" in pod \"image-registry-597d5ff6c5-zq2s2\" is waiting to start: ContainerCreating" I0704 06:58:35.402879 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for registry container image-registry-6bb59d478d-7jbv2 pod in namespace openshift-image-registry (previous: false). I0704 06:58:35.591561 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0704 06:58:35.591631 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-5fsc8 pod in namespace openshift-image-registry (previous: false). I0704 06:58:35.795812 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0704 06:58:35.795839 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-ccgqt pod in namespace openshift-image-registry (previous: false). I0704 06:58:36.010326 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0704 06:58:36.010346 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for node-ca container node-ca-ckq5n pod in namespace openshift-image-registry (previous: false). I0704 06:58:36.199092 1 gather_cluster_operator_pods_and_events.go:280] Error: "log buffer is empty" I0704 06:58:36.199119 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-7477d86bcd-lj7cd pod in namespace openshift-ingress (previous: false). W0704 06:58:36.322829 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. I0704 06:58:36.401770 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for router-default-7477d86bcd-lj7cd pod in namespace openshift-ingress for failing operator router (previous: false): "container \"router\" in pod \"router-default-7477d86bcd-lj7cd\" is waiting to start: ContainerCreating" I0704 06:58:36.401794 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"router\" in pod \"router-default-7477d86bcd-lj7cd\" is waiting to start: ContainerCreating" I0704 06:58:36.401817 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-7477d86bcd-w654p pod in namespace openshift-ingress (previous: false). I0704 06:58:36.596513 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for router-default-7477d86bcd-w654p pod in namespace openshift-ingress for failing operator router (previous: false): "container \"router\" in pod \"router-default-7477d86bcd-w654p\" is waiting to start: ContainerCreating" I0704 06:58:36.596533 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"router\" in pod \"router-default-7477d86bcd-w654p\" is waiting to start: ContainerCreating" I0704 06:58:36.596544 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for router container router-default-76bd8cc569-b48rn pod in namespace openshift-ingress (previous: false). I0704 06:58:36.796052 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for router-default-76bd8cc569-b48rn pod in namespace openshift-ingress for failing operator router (previous: false): "container \"router\" in pod \"router-default-76bd8cc569-b48rn\" is waiting to start: ContainerCreating" I0704 06:58:36.796076 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"router\" in pod \"router-default-76bd8cc569-b48rn\" is waiting to start: ContainerCreating" I0704 06:58:36.796090 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-5cqjx pod in namespace openshift-ingress-canary (previous: false). I0704 06:58:36.996155 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for ingress-canary-5cqjx pod in namespace openshift-ingress-canary for failing operator serve-healthcheck-canary (previous: false): "container \"serve-healthcheck-canary\" in pod \"ingress-canary-5cqjx\" is waiting to start: ContainerCreating" I0704 06:58:36.996175 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"serve-healthcheck-canary\" in pod \"ingress-canary-5cqjx\" is waiting to start: ContainerCreating" I0704 06:58:36.996186 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for serve-healthcheck-canary container ingress-canary-sxb86 pod in namespace openshift-ingress-canary (previous: false). I0704 06:58:37.195860 1 gather_cluster_operator_pods_and_events.go:408] Failed to fetch log for ingress-canary-sxb86 pod in namespace openshift-ingress-canary for failing operator serve-healthcheck-canary (previous: false): "container \"serve-healthcheck-canary\" in pod \"ingress-canary-sxb86\" is waiting to start: ContainerCreating" I0704 06:58:37.195880 1 gather_cluster_operator_pods_and_events.go:280] Error: "container \"serve-healthcheck-canary\" in pod \"ingress-canary-sxb86\" is waiting to start: ContainerCreating" I0704 06:58:37.195892 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for migrator container migrator-7d5f866c57-dt72r pod in namespace openshift-kube-storage-version-migrator (previous: false). W0704 06:58:37.328654 1 gather_dvo_metrics.go:210] Failed to read the DVO metrics. Trying again. W0704 06:58:37.328685 1 gather_dvo_metrics.go:117] Unable to read metrics from endpoint "http://deployment-validation-operator-metrics.openshift-deployment-validation-operator.svc:8383": DVO metrics service was not available within the 5s timeout: context deadline exceeded I0704 06:58:37.328725 1 tasks_processing.go:74] worker 8 stopped. E0704 06:58:37.328737 1 gather.go:140] gatherer "clusterconfig" function "dvo_metrics" failed with the error: DVO metrics service was not available within the 5s timeout: context deadline exceeded I0704 06:58:37.328749 1 recorder.go:75] Recording config/dvo_metrics with fingerprint=e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 W0704 06:58:37.328765 1 gather.go:155] issue recording gatherer "clusterconfig" function "dvo_metrics" result "config/dvo_metrics" because of the warning: warning: the record with the same fingerprint "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" was already recorded at path "config/configmaps/openshift-config/openshift-service-ca.crt/service-ca.crt", recording another one with a different path "config/dvo_metrics" I0704 06:58:37.328791 1 gather.go:177] gatherer "clusterconfig" function "dvo_metrics" took 5.065984659s to process 1 records I0704 06:58:37.395223 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for graceful-termination container migrator-7d5f866c57-dt72r pod in namespace openshift-kube-storage-version-migrator (previous: false). I0704 06:58:37.596098 1 gather_cluster_operator_pods_and_events.go:365] Fetching logs for kube-storage-version-migrator-operator container kube-storage-version-migrator-operator-74848b4cb9-xm77b pod in namespace openshift-kube-storage-version-migrator-operator (previous: false). I0704 06:58:37.798347 1 tasks_processing.go:74] worker 24 stopped. I0704 06:58:37.798455 1 recorder.go:75] Recording events/openshift-dns-operator with fingerprint=cc2ff4a27522b55cf20a22678800a3b775617ca21923f546a152d8315f7d46be I0704 06:58:37.798502 1 recorder.go:75] Recording events/openshift-dns with fingerprint=7634e042318f37def0d499686ed38d1214a068ba4ff9fc13dae850d6d7b34b6e I0704 06:58:37.798586 1 recorder.go:75] Recording events/openshift-image-registry with fingerprint=0b743e8f21a9bcf7b9adb1c2a7df9dc8e587c64b6c469ff4461f3df77b74b0ba I0704 06:58:37.798612 1 recorder.go:75] Recording events/openshift-ingress-operator with fingerprint=f34dc6a91dd8d99ef9592b71d39fbea75976404626d59cc33598401ea4ececf0 I0704 06:58:37.798657 1 recorder.go:75] Recording events/openshift-ingress with fingerprint=8c464c6473dff431911626d60aa3057961bcc8751d5a10a5aab156ec1afa5430 I0704 06:58:37.798671 1 recorder.go:75] Recording events/openshift-ingress-canary with fingerprint=3465a5b605e9e47cf9a24a2927da4ffc2dc2c310f940c0d3d3a0a072c1f260f7 I0704 06:58:37.798680 1 recorder.go:75] Recording events/openshift-kube-storage-version-migrator with fingerprint=3ac27a1a330f97640451891aea00ff9cee06c5e1a0241ef5c4177d6f9efb8d37 I0704 06:58:37.798730 1 recorder.go:75] Recording events/openshift-kube-storage-version-migrator-operator with fingerprint=d48b184999818dae0fdc0fc5510a98ec323e9ac84ebbc1862f6017b1c383db62 I0704 06:58:37.798744 1 recorder.go:75] Recording config/pod/openshift-kube-storage-version-migrator/logs/migrator-7d5f866c57-dt72r/migrator_current.log with fingerprint=1776003be6037b6ec758f2a3c3b8f734ef30bd2cf66816a371200bbb930852d6 I0704 06:58:37.798750 1 recorder.go:75] Recording config/pod/openshift-kube-storage-version-migrator/logs/migrator-7d5f866c57-dt72r/graceful-termination_current.log with fingerprint=9d2d5558176efa65b89a33c19f99fa31b8432942fe379c98915d815ad85218fd I0704 06:58:37.798814 1 recorder.go:75] Recording config/pod/openshift-kube-storage-version-migrator-operator/logs/kube-storage-version-migrator-operator-74848b4cb9-xm77b/kube-storage-version-migrator-operator_current.log with fingerprint=e2973569d7884a7245256cca17683740202a4fc1db2cc64a27928975f30694c8 I0704 06:58:37.798825 1 gather.go:177] gatherer "clusterconfig" function "operators_pods_and_events" took 5.517670111s to process 11 records I0704 06:58:44.699725 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 06:58:44.897518 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 06:58:44.937251 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 06:58:45.403895 1 tasks_processing.go:74] worker 12 stopped. I0704 06:58:45.403933 1 recorder.go:75] Recording config/installplans with fingerprint=7b887df561a3a9e6ef0dc672845aa5d56e348505006b7496d3a2f83892b0c95b I0704 06:58:45.403947 1 gather.go:177] gatherer "clusterconfig" function "install_plans" took 13.084700575s to process 1 records I0704 06:58:46.077437 1 tasks_processing.go:74] worker 6 stopped. I0704 06:58:46.077786 1 recorder.go:75] Recording config/serviceaccounts with fingerprint=3a23a80037b37537e6bce19267a0686640cc3d2d84bcc6f7bc973979a947d17d I0704 06:58:46.077808 1 gather.go:177] gatherer "clusterconfig" function "service_accounts" took 13.806148815s to process 1 records E0704 06:58:46.077876 1 periodic.go:247] "Unhandled Error" err="clusterconfig failed after 13.818s with: function \"pod_network_connectivity_checks\" failed with an error, function \"support_secret\" failed with an error, function \"machines\" failed with an error, function \"machine_healthchecks\" failed with an error, function \"config_maps\" failed with an error, function \"ingress_certificates\" failed with an error, function \"dvo_metrics\" failed with an error" I0704 06:58:46.079001 1 controllerstatus.go:89] name=periodic-clusterconfig healthy=false reason=PeriodicGatherFailed message=Source clusterconfig could not be retrieved: function "pod_network_connectivity_checks" failed with an error, function "support_secret" failed with an error, function "machines" failed with an error, function "machine_healthchecks" failed with an error, function "config_maps" failed with an error, function "ingress_certificates" failed with an error, function "dvo_metrics" failed with an error I0704 06:58:46.079018 1 periodic.go:209] Running workloads gatherer I0704 06:58:46.079034 1 tasks_processing.go:45] number of workers: 2 I0704 06:58:46.079043 1 tasks_processing.go:69] worker 1 listening for tasks. I0704 06:58:46.079048 1 tasks_processing.go:71] worker 1 working on helmchart_info task. I0704 06:58:46.079060 1 tasks_processing.go:69] worker 0 listening for tasks. I0704 06:58:46.079129 1 tasks_processing.go:71] worker 0 working on workload_info task. I0704 06:58:46.106774 1 tasks_processing.go:74] worker 1 stopped. I0704 06:58:46.106794 1 gather.go:177] gatherer "workloads" function "helmchart_info" took 27.710406ms to process 0 records I0704 06:58:46.198929 1 gather_workloads_info.go:278] Loaded pods in 0s, will wait 22s for image data I0704 06:58:46.207048 1 gather_workloads_info.go:387] No image sha256:80748ba08e1c264a8c105e7f607eff386a66378e024443a844993ee9292858c1 (9ms) I0704 06:58:46.218109 1 gather_workloads_info.go:387] No image sha256:2904a78e2eb73fd6a9bb94c105c2a056831fb4113fbb7b0607c50adc9d879c9b (11ms) I0704 06:58:46.225981 1 gather_workloads_info.go:387] No image sha256:04c87c054a3f366a7dfbe0a93ebb0c80a098ee16842c5794b67c1202eec61996 (8ms) I0704 06:58:46.234052 1 gather_workloads_info.go:387] No image sha256:765f0d23b637f685f98a31bd47c131b03cf72a40761a3f9a9d6320faa3c33733 (8ms) I0704 06:58:46.242187 1 gather_workloads_info.go:387] No image sha256:289816958633a763a72dbc44e1dad40466223164e7e253039514f0d974ea5d21 (8ms) I0704 06:58:46.250300 1 gather_workloads_info.go:387] No image sha256:a258c226562adb14e3a163a1940938526ee6a0928982a7667d85d9a7334ce639 (8ms) I0704 06:58:46.257887 1 gather_workloads_info.go:387] No image sha256:521712486e2c6e3c020dad6a1cb340db8e55665b69f7c208fab9cd9e965fd588 (8ms) I0704 06:58:46.265543 1 gather_workloads_info.go:387] No image sha256:fac526ab9335a9a8decb028a30673f4ae718deae8da114f36103b6438ba6458f (8ms) I0704 06:58:46.273238 1 gather_workloads_info.go:387] No image sha256:95418d0ba6fc0aa96647d97cf6530967f7d7ae97a82ce7f7e41c8e44310b0b72 (8ms) I0704 06:58:46.283135 1 gather_workloads_info.go:387] No image sha256:c940ea87e7d133d75ba0002ef00c0806825eed3db8094cdb260d1bac18127733 (10ms) I0704 06:58:46.306331 1 gather_workloads_info.go:387] No image sha256:56a85660a445eced5c79a595a0eccf590087c5672d50f49d4c25ad52f9a44f04 (23ms) I0704 06:58:46.407541 1 gather_workloads_info.go:387] No image sha256:1a2532940843248c57d52141185dd71fbc393ab28b65d48f682038632c1dbbad (101ms) I0704 06:58:46.511310 1 gather_workloads_info.go:387] No image sha256:7adc1eab05d6724c76ba751f6df816b08d6e70b78dee9eb94fa6fd9690542c98 (104ms) I0704 06:58:46.608086 1 gather_workloads_info.go:387] No image sha256:2598489729a4b258e4ecda4a06f6875133f2a10ced5c5241f8a57a8a05418e36 (97ms) I0704 06:58:46.707231 1 gather_workloads_info.go:387] No image sha256:943018739e3db1763c3184b460dbc409e058abbac76d57b9927faad317be85e4 (99ms) I0704 06:58:46.810173 1 gather_workloads_info.go:387] No image sha256:2e564f336c77116053f34d4201d364d8da04e789cfffa0ea422574c95f2d6404 (103ms) I0704 06:58:46.907392 1 gather_workloads_info.go:387] No image sha256:36b9e89c3cfcf1ab9ae500486e38afb6862cba48cb0b4d84a09508ab8f3d299f (97ms) I0704 06:58:47.007749 1 gather_workloads_info.go:387] No image sha256:695cf2f0cc07683c2a3ce1eaf3e56fe18abc6e2bac716f7d9843f5d173b9df52 (100ms) I0704 06:58:47.107998 1 gather_workloads_info.go:387] No image sha256:03cf4cd7ef1518610c6c7b3ad27d1622d82e98e3dc6e3f8e5d0fceb5c8d3786e (100ms) I0704 06:58:47.208323 1 gather_workloads_info.go:387] No image sha256:7b31223098f08328f5ddea8e5b871dbbd5f5a61ec550e8956f66793c0c6031a9 (100ms) I0704 06:58:47.308319 1 gather_workloads_info.go:387] No image sha256:91828234f107c068c8a4966d08370ae7b73e637651dbc6d92c18c4553402c22c (100ms) I0704 06:58:47.412920 1 gather_workloads_info.go:387] No image sha256:2e57e192c3c1240fd935dcd55c8fde5e70e78bf81d6176c96edf21fafe59f8ba (105ms) I0704 06:58:47.508156 1 gather_workloads_info.go:387] No image sha256:0a99240166165eb5718e7516a43282fe32df9c7c5e809b31b58abe44e42ff94d (95ms) I0704 06:58:47.607581 1 gather_workloads_info.go:387] No image sha256:a498046d64605bcccee2440aa4f04a4602baaae263cf01d977ec5208e876b1fd (99ms) I0704 06:58:47.707590 1 gather_workloads_info.go:387] No image sha256:a56211d075aa43cbb491f669a5b2e46ee023dc95b7d51dbac28f463948c5ad61 (100ms) I0704 06:58:47.808088 1 gather_workloads_info.go:387] No image sha256:5a95c19d82767e0235b4edb4a0536482c816904897aae1dc3eb255cb52b87a9f (100ms) I0704 06:58:47.907278 1 gather_workloads_info.go:387] No image sha256:653c666f842c13e0baae2e89a9b1efe0e2ef56f621ffb5b32005115d2a26ab8c (99ms) I0704 06:58:48.007696 1 gather_workloads_info.go:387] No image sha256:47154813651033d59751fb655a384dbffb64dd26f10bd7f3be0c3128d0486356 (100ms) I0704 06:58:48.115469 1 gather_workloads_info.go:387] No image sha256:b3909bf664c77097f75b3768830863d642eed3815dab2bfb4415c771ca2d5007 (108ms) I0704 06:58:48.207655 1 gather_workloads_info.go:387] No image sha256:ca1344cb64140188b7cae7bbc51fb751566c0b0c97d5e39b5850e628032c4a5e (92ms) I0704 06:58:48.308196 1 gather_workloads_info.go:387] No image sha256:5f0b67cfbbc381243fb91ccc17345b56d05f4d717c667e8c644e5bf05633ba71 (101ms) I0704 06:58:48.408574 1 gather_workloads_info.go:387] No image sha256:4556896f77307821531ef91b7b7faccb82b824ea695693b2989f597f0deca038 (100ms) I0704 06:58:48.507811 1 gather_workloads_info.go:387] No image sha256:e84cb128d930bd1ab867cc89b7b7bf2b2c0e41105ab93b5381069945b3ee9c57 (99ms) I0704 06:58:48.610426 1 gather_workloads_info.go:387] No image sha256:a0105d1eb62cf6ac9e5e2ef28d3e89bf6dc514bc594fc7090fe5a5ee18a09c87 (103ms) I0704 06:58:48.610459 1 tasks_processing.go:74] worker 0 stopped. E0704 06:58:48.610470 1 gather.go:140] gatherer "workloads" function "workload_info" failed with the error: no running pods found for the insights-runtime-extractor statefulset I0704 06:58:48.610802 1 recorder.go:75] Recording config/workload_info with fingerprint=ad044d41e5f5beb90725462c69b8bd9d0fe03fb40448ba7b45b6ee780f1b8796 I0704 06:58:48.610818 1 gather.go:177] gatherer "workloads" function "workload_info" took 2.531317699s to process 1 records E0704 06:58:48.610842 1 periodic.go:247] "Unhandled Error" err="workloads failed after 2.531s with: function \"workload_info\" failed with an error" I0704 06:58:48.612487 1 controllerstatus.go:89] name=periodic-workloads healthy=false reason=PeriodicGatherFailed message=Source workloads could not be retrieved: function "workload_info" failed with an error I0704 06:58:48.612501 1 periodic.go:209] Running conditional gatherer I0704 06:58:48.618847 1 requests.go:294] Making HTTP GET request at: https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules I0704 06:58:48.625499 1 conditional_gatherer.go:107] Get "https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules": dial tcp: lookup console.redhat.com on 172.30.0.10:53: read udp 10.128.0.14:53827->172.30.0.10:53: read: connection refused E0704 06:58:48.625768 1 conditional_gatherer.go:322] unable to update alerts cache: open /var/run/configmaps/service-ca-bundle/service-ca.crt: no such file or directory I0704 06:58:48.625842 1 conditional_gatherer.go:384] updating version cache for conditional gatherer I0704 06:58:48.631915 1 conditional_gatherer.go:392] cluster version is '4.20.8' E0704 06:58:48.631930 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 06:58:48.631935 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 06:58:48.631938 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 06:58:48.631941 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 06:58:48.631944 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 06:58:48.631947 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 06:58:48.631950 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 06:58:48.631953 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing E0704 06:58:48.631956 1 conditional_gatherer.go:209] error checking conditions for a gathering rule: alerts cache is missing I0704 06:58:48.631989 1 tasks_processing.go:45] number of workers: 3 I0704 06:58:48.632013 1 tasks_processing.go:69] worker 2 listening for tasks. I0704 06:58:48.632020 1 tasks_processing.go:71] worker 2 working on conditional_gatherer_rules task. I0704 06:58:48.632022 1 tasks_processing.go:69] worker 0 listening for tasks. I0704 06:58:48.632039 1 tasks_processing.go:69] worker 1 listening for tasks. I0704 06:58:48.632048 1 tasks_processing.go:74] worker 1 stopped. I0704 06:58:48.632055 1 tasks_processing.go:71] worker 2 working on rapid_container_logs task. I0704 06:58:48.632073 1 tasks_processing.go:71] worker 0 working on remote_configuration task. I0704 06:58:48.632120 1 recorder.go:75] Recording insights-operator/conditional-gatherer-rules with fingerprint=7034af97e7e41c22e4b775abdd4b9066c8ebb19da33eb7f69f39bfd2eb5f6406 I0704 06:58:48.632134 1 gather.go:177] gatherer "conditional" function "conditional_gatherer_rules" took 1.198µs to process 1 records I0704 06:58:48.632172 1 recorder.go:75] Recording insights-operator/remote-configuration with fingerprint=0394430c431eec4d48bb1811a90918e95161d2282c59af26f2473613cc0959db I0704 06:58:48.632181 1 gather.go:177] gatherer "conditional" function "remote_configuration" took 3.804µs to process 1 records I0704 06:58:48.632187 1 tasks_processing.go:74] worker 0 stopped. I0704 06:58:48.632334 1 tasks_processing.go:74] worker 2 stopped. I0704 06:58:48.632349 1 gather.go:177] gatherer "conditional" function "rapid_container_logs" took 272.51µs to process 0 records I0704 06:58:48.632376 1 controllerstatus.go:89] name=periodic-conditional healthy=false reason=NotAvailable message=Get "https://console.redhat.com/api/gathering/v2/4.20.8/gathering_rules": dial tcp: lookup console.redhat.com on 172.30.0.10:53: read udp 10.128.0.14:53827->172.30.0.10:53: read: connection refused I0704 06:58:48.632402 1 recorder.go:75] Recording insights-operator/remote-configuration.json with fingerprint=359de9c990c741675cec72fda96b5c3682221efdb4799f5eaa6e9805bcd3b5c1 I0704 06:58:48.653235 1 recorder.go:75] Recording insights-operator/gathers with fingerprint=222c0b4aa64e8bc2b7231c6c3da61fdba2d190e72b19579deadf8a20cc025736 I0704 06:58:48.653370 1 diskrecorder.go:70] Writing 107 records to /var/lib/insights-operator/insights-2026-07-04-065848.tar.gz I0704 06:58:48.660365 1 diskrecorder.go:51] Wrote 107 records to disk in 7ms I0704 06:58:48.660398 1 periodic.go:278] Gathering cluster info every 2h0m0s I0704 06:58:48.660415 1 periodic.go:279] Configuration is dataReporting: interval: 2h0m0s, uploadEndpoint: https://console.redhat.com/api/ingress/v1/upload, storagePath: /var/lib/insights-operator, downloadEndpoint: https://console.redhat.com/api/insights-results-aggregator/v2/cluster/%s/reports, conditionalGathererEndpoint: https://console.redhat.com/api/gathering/v2/%s/gathering_rules, obfuscation: [] sca: disabled: false, endpoint: https://api.openshift.com/api/accounts_mgmt/v1/entitlement_certificates, interval: 8h0m0s alerting: disabled: false clusterTransfer: endpoint: https://api.openshift.com/api/accounts_mgmt/v1/cluster_transfers/, interval: 12h0m0s proxy: httpProxy: , httpsProxy: , noProxy: I0704 06:59:00.279349 1 configmapobserver.go:84] configmaps "insights-config" not found I0704 06:59:36.577079 1 observer_polling.go:111] Observed file "/var/run/secrets/serving-cert/tls.crt" has been created (hash="9ea776376c2e44d6dd032b2ca5b0a817c9bf109474aa6ba99d97fe8c0aaaa292") W0704 06:59:36.577112 1 builder.go:160] Restart triggered because of file /var/run/secrets/serving-cert/tls.crt was created I0704 06:59:36.577167 1 observer_polling.go:111] Observed file "/var/run/secrets/serving-cert/tls.key" has been created (hash="5f06881ebbb42deebdf174625787854731fed0fb9d3c025c1a83b1788e14f61f") I0704 06:59:36.577187 1 genericapiserver.go:548] "[graceful-termination] shutdown event" name="ShutdownInitiated" I0704 06:59:36.577215 1 genericapiserver.go:693] "[graceful-termination] pre-shutdown hooks completed" name="PreShutdownHooksStopped" I0704 06:59:36.577217 1 base_controller.go:181] Shutting down ConfigController ... I0704 06:59:36.577251 1 simple_featuregate_reader.go:177] Shutting down feature-gate-detector I0704 06:59:36.577254 1 base_controller.go:123] Shutting down worker of ConfigController controller ... I0704 06:59:36.577262 1 base_controller.go:113] All ConfigController workers have been terminated I0704 06:59:36.577271 1 base_controller.go:181] Shutting down LoggingSyncer ... I0704 06:59:36.577278 1 genericapiserver.go:651] "[graceful-termination] not going to wait for active watch request(s) to drain" I0704 06:59:36.577298 1 object_count_tracker.go:151] "StorageObjectCountTracker pruner is exiting" I0704 06:59:36.577311 1 periodic.go:170] Shutting down I0704 06:59:36.577318 1 dynamic_serving_content.go:149] "Shutting down controller" name="serving-cert::/tmp/serving-cert-2425336428/tls.crt::/tmp/serving-cert-2425336428/tls.key" I0704 06:59:36.577323 1 base_controller.go:123] Shutting down worker of LoggingSyncer controller ... I0704 06:59:36.577330 1 base_controller.go:113] All LoggingSyncer workers have been terminated