time=2026-05-07T15:35:27.391Z level=INFO source=main.go:274 msg="Experimental delayed compaction is enabled."
time=2026-05-07T15:35:27.448Z level=INFO source=main.go:1547 msg="updated GOGC" old=100 new=75
time=2026-05-07T15:35:27.448Z level=INFO source=main.go:679 msg="Leaving GOMAXPROCS=16: CPU quota undefined" component=automaxprocs
time=2026-05-07T15:35:27.448Z level=INFO source=memlimit.go:198 msg="GOMEMLIMIT is updated" component=automemlimit package=github.com/KimMachineGun/automemlimit/memlimit GOMEMLIMIT=59290096435 previous=9223372036854775807
time=2026-05-07T15:35:27.448Z level=INFO source=main.go:772 msg="Starting Prometheus Server" mode=server version="(version=3.5.0, branch=HEAD, revision=5ba72b05c658bebd2be9b22c593cfba886812422)"
time=2026-05-07T15:35:27.448Z level=INFO source=main.go:777 msg="operational information" build_context="(go=go1.24.6 (Red Hat 1.24.6-1.el9_6) X:strictfipsruntime, platform=linux/amd64, user=reproducible@reproducible, date=20250929-09:52:01, tags=netgo,builtinassets,strictfipsruntime)" host_details="(Linux 5.14.0-570.73.1.el9_6.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Dec 4 01:12:45 EST 2025 x86_64 prometheus-k8s-0 (none))" fd_limits="(soft=1048576, hard=1048576)" vm_limits="(soft=unlimited, hard=unlimited)"
time=2026-05-07T15:35:27.450Z level=INFO source=web.go:656 msg="Start listening for connections" component=web address=127.0.0.1:9090
time=2026-05-07T15:35:27.450Z level=INFO source=main.go:1291 msg="Starting TSDB ..."
time=2026-05-07T15:35:27.454Z level=INFO source=tls_config.go:347 msg="Listening on" component=web address=127.0.0.1:9090
time=2026-05-07T15:35:27.454Z level=INFO source=tls_config.go:386 msg="TLS is disabled." component=web http2=false address=127.0.0.1:9090
time=2026-05-07T15:35:27.455Z level=INFO source=head.go:657 msg="Replaying on-disk memory mappable chunks if any" component=tsdb
time=2026-05-07T15:35:27.455Z level=INFO source=head.go:744 msg="On-disk memory mappable chunks replay completed" component=tsdb duration=1.307µs
time=2026-05-07T15:35:27.455Z level=INFO source=head.go:752 msg="Replaying WAL, this may take a while" component=tsdb
time=2026-05-07T15:35:27.455Z level=INFO source=head.go:825 msg="WAL segment loaded" component=tsdb segment=0 maxSegment=0 duration=369.797µs
time=2026-05-07T15:35:27.455Z level=INFO source=head.go:862 msg="WAL replay completed" component=tsdb checkpoint_replay_duration=27.223µs wal_replay_duration=391.329µs wbl_replay_duration=172ns chunk_snapshot_load_duration=0s mmap_chunk_replay_duration=1.307µs total_replay_duration=467.001µs
time=2026-05-07T15:35:27.458Z level=INFO source=main.go:1312 msg="filesystem information" fs_type=XFS_SUPER_MAGIC
time=2026-05-07T15:35:27.458Z level=INFO source=main.go:1315 msg="TSDB started"
time=2026-05-07T15:35:27.458Z level=INFO source=main.go:1500 msg="Loading configuration file" filename=/etc/prometheus/config_out/prometheus.env.yaml
time=2026-05-07T15:35:27.472Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-cluster-machine-approver/cluster-machine-approver/0
time=2026-05-07T15:35:27.473Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-kube-controller-manager-operator/kube-controller-manager-operator/0
time=2026-05-07T15:35:27.473Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-monitoring/kubelet/1
time=2026-05-07T15:35:27.473Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-monitoring/thanos-sidecar/0
time=2026-05-07T15:35:27.474Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-ingress-operator/ingress-operator/0
time=2026-05-07T15:35:27.474Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-cluster-storage-operator/cluster-storage-operator/0
time=2026-05-07T15:35:27.474Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-console/console/0
time=2026-05-07T15:35:27.475Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-kube-apiserver/openshift-kube-apiserver/0
time=2026-05-07T15:35:27.475Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-network-diagnostics/network-check-source/0
time=2026-05-07T15:35:27.475Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-ovn-kubernetes/monitor-ovn-node/1
time=2026-05-07T15:35:27.475Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-cluster-node-tuning-operator/node-tuning-operator/0
time=2026-05-07T15:35:27.476Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-cluster-samples-operator/cluster-samples-operator/0
time=2026-05-07T15:35:27.476Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-console-operator/console-operator/0
time=2026-05-07T15:35:27.476Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-ingress/router-default/0
time=2026-05-07T15:35:27.477Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-multus/monitor-network/0
time=2026-05-07T15:35:27.477Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-service-ca-operator/service-ca-operator/0
time=2026-05-07T15:35:27.477Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-authentication-operator/authentication-operator/0
time=2026-05-07T15:35:27.478Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-cloud-credential-operator/cloud-credential-operator/0
time=2026-05-07T15:35:27.478Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-config-operator/config-operator/0
time=2026-05-07T15:35:27.478Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-dns-operator/dns-operator/0
time=2026-05-07T15:35:27.478Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-network-operator/network-operator/0
time=2026-05-07T15:35:27.479Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-dns/dns-default/0
time=2026-05-07T15:35:27.479Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-image-registry/image-registry/0
time=2026-05-07T15:35:27.480Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-apiserver-operator/openshift-apiserver-operator/0
time=2026-05-07T15:35:27.480Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-insights/insights-operator/0
time=2026-05-07T15:35:27.481Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager notify" discovery=kubernetes config=config-0
time=2026-05-07T15:35:27.562Z level=INFO source=main.go:1540 msg="Completed loading of configuration file" db_storage=1.572µs remote_storage=1.446µs web_handler=886ns query_engine=1.273µs scrape=242.943µs scrape_sd=8.568856ms notify=341.931µs notify_sd=1.512541ms rules=79.733325ms tracing=5.257µs filename=/etc/prometheus/config_out/prometheus.env.yaml totalDuration=104.516647ms
time=2026-05-07T15:35:27.562Z level=INFO source=main.go:1276 msg="Server is ready to receive web requests."
time=2026-05-07T15:35:27.562Z level=INFO source=manager.go:176 msg="Starting rule manager..." component="rule manager"
time=2026-05-07T15:35:27.572Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.572Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.572Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.572Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.573Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.573Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.573Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.573Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.574Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.575Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.575Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.575Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.575Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.575Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.575Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.575Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.576Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.576Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.576Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.576Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.577Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.577Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.577Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.577Z level=INFO source=reflector.go:569 msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:503: failed to list *v1.Service: services is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"services\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\"" component=k8s_client_runtime
time=2026-05-07T15:35:27.577Z level=ERROR source=reflector.go:166 msg="Unhandled Error" component=k8s_client_runtime logger=UnhandledError err="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:503: Failed to watch *v1.Service: failed to list *v1.Service: services is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"services\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\""
time=2026-05-07T15:35:27.577Z level=INFO source=reflector.go:569 msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:502: failed to list *v1.Endpoints: endpoints is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"endpoints\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\"" component=k8s_client_runtime
time=2026-05-07T15:35:27.577Z level=ERROR source=reflector.go:166 msg="Unhandled Error" component=k8s_client_runtime logger=UnhandledError err="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:502: Failed to watch *v1.Endpoints: failed to list *v1.Endpoints: endpoints is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"endpoints\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\""
time=2026-05-07T15:35:27.578Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.578Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.578Z level=INFO source=reflector.go:569 msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:504: failed to list *v1.Pod: pods is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"pods\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\"" component=k8s_client_runtime
time=2026-05-07T15:35:27.578Z level=ERROR source=reflector.go:166 msg="Unhandled Error" component=k8s_client_runtime logger=UnhandledError err="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:504: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"pods\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\""
time=2026-05-07T15:35:27.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.587Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.587Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.587Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.590Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.590Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.590Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.590Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.598Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.598Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.598Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.598Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.598Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.598Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:27.598Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:28.582Z level=INFO source=reflector.go:569 msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:502: failed to list *v1.Endpoints: endpoints is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"endpoints\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\"" component=k8s_client_runtime
time=2026-05-07T15:35:28.582Z level=ERROR source=reflector.go:166 msg="Unhandled Error" component=k8s_client_runtime logger=UnhandledError err="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:502: Failed to watch *v1.Endpoints: failed to list *v1.Endpoints: endpoints is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"endpoints\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\""
time=2026-05-07T15:35:28.720Z level=INFO source=reflector.go:569 msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:504: failed to list *v1.Pod: pods is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"pods\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\"" component=k8s_client_runtime
time=2026-05-07T15:35:28.720Z level=ERROR source=reflector.go:166 msg="Unhandled Error" component=k8s_client_runtime logger=UnhandledError err="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:504: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"pods\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\""
time=2026-05-07T15:35:28.901Z level=INFO source=reflector.go:569 msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:503: failed to list *v1.Service: services is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"services\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\"" component=k8s_client_runtime
time=2026-05-07T15:35:28.901Z level=ERROR source=reflector.go:166 msg="Unhandled Error" component=k8s_client_runtime logger=UnhandledError err="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:503: Failed to watch *v1.Service: failed to list *v1.Service: services is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"services\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\""
time=2026-05-07T15:35:31.507Z level=INFO source=reflector.go:569 msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:503: failed to list *v1.Service: services is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"services\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\"" component=k8s_client_runtime
time=2026-05-07T15:35:31.507Z level=ERROR source=reflector.go:166 msg="Unhandled Error" component=k8s_client_runtime logger=UnhandledError err="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:503: Failed to watch *v1.Service: failed to list *v1.Service: services is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"services\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\""
time=2026-05-07T15:35:31.609Z level=INFO source=reflector.go:569 msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:502: failed to list *v1.Endpoints: endpoints is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"endpoints\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\"" component=k8s_client_runtime
time=2026-05-07T15:35:31.609Z level=ERROR source=reflector.go:166 msg="Unhandled Error" component=k8s_client_runtime logger=UnhandledError err="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:502: Failed to watch *v1.Endpoints: failed to list *v1.Endpoints: endpoints is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"endpoints\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\""
time=2026-05-07T15:35:31.816Z level=INFO source=reflector.go:569 msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:504: failed to list *v1.Pod: pods is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"pods\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\"" component=k8s_client_runtime
time=2026-05-07T15:35:31.816Z level=ERROR source=reflector.go:166 msg="Unhandled Error" component=k8s_client_runtime logger=UnhandledError err="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:504: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"pods\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\""
time=2026-05-07T15:35:32.453Z level=INFO source=main.go:1500 msg="Loading configuration file" filename=/etc/prometheus/config_out/prometheus.env.yaml
time=2026-05-07T15:35:32.535Z level=INFO source=main.go:1540 msg="Completed loading of configuration file" db_storage=1.144µs remote_storage=2.09µs web_handler=564ns query_engine=1.152µs scrape=75.186µs scrape_sd=512.796µs notify=409.935µs notify_sd=27.209µs rules=68.993341ms tracing=5.821µs filename=/etc/prometheus/config_out/prometheus.env.yaml totalDuration=82.025273ms
time=2026-05-07T15:35:35.642Z level=INFO source=reflector.go:569 msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:504: failed to list *v1.Pod: pods is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"pods\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\"" component=k8s_client_runtime
time=2026-05-07T15:35:35.642Z level=ERROR source=reflector.go:166 msg="Unhandled Error" component=k8s_client_runtime logger=UnhandledError err="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:504: Failed to watch *v1.Pod: failed to list *v1.Pod: pods is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"pods\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\""
time=2026-05-07T15:35:36.469Z level=INFO source=reflector.go:569 msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:503: failed to list *v1.Service: services is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"services\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\"" component=k8s_client_runtime
time=2026-05-07T15:35:36.469Z level=ERROR source=reflector.go:166 msg="Unhandled Error" component=k8s_client_runtime logger=UnhandledError err="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:503: Failed to watch *v1.Service: failed to list *v1.Service: services is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"services\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\""
time=2026-05-07T15:35:37.973Z level=INFO source=reflector.go:569 msg="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:502: failed to list *v1.Endpoints: endpoints is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"endpoints\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\"" component=k8s_client_runtime
time=2026-05-07T15:35:37.973Z level=ERROR source=reflector.go:166 msg="Unhandled Error" component=k8s_client_runtime logger=UnhandledError err="github.com/prometheus/prometheus/discovery/kubernetes/kubernetes.go:502: Failed to watch *v1.Endpoints: failed to list *v1.Endpoints: endpoints is forbidden: User \"system:serviceaccount:openshift-monitoring:prometheus-k8s\" cannot list resource \"endpoints\" in API group \"\" in the namespace \"openshift-cluster-samples-operator\""
time=2026-05-07T15:35:45.811Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:35:45.813Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:36:18.139Z level=INFO source=main.go:1500 msg="Loading configuration file" filename=/etc/prometheus/config_out/prometheus.env.yaml
time=2026-05-07T15:36:18.219Z level=INFO source=main.go:1540 msg="Completed loading of configuration file" db_storage=948ns remote_storage=1.467µs web_handler=878ns query_engine=1.137µs scrape=2.503098ms scrape_sd=541.105µs notify=376.857µs notify_sd=11.159µs rules=64.412988ms tracing=6.444µs filename=/etc/prometheus/config_out/prometheus.env.yaml totalDuration=80.87543ms
time=2026-05-07T15:37:30.344Z level=INFO source=main.go:1500 msg="Loading configuration file" filename=/etc/prometheus/config_out/prometheus.env.yaml
time=2026-05-07T15:37:30.443Z level=INFO source=main.go:1540 msg="Completed loading of configuration file" db_storage=1.38µs remote_storage=1.746µs web_handler=483ns query_engine=1.079µs scrape=2.135099ms scrape_sd=552.333µs notify=346.752µs notify_sd=10.763µs rules=83.188623ms tracing=6.171µs filename=/etc/prometheus/config_out/prometheus.env.yaml totalDuration=99.176722ms
time=2026-05-07T15:40:27.593Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:40:37.583Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:40:42.600Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:40:49.591Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:41:11.591Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:41:15.594Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:41:22.583Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:41:26.584Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:41:27.590Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:42:06.598Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:42:17.590Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:42:57.737Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:43:03.590Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:43:14.584Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:43:20.595Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:43:40.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:43:42.584Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:43:43.593Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:44:07.596Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:44:10.592Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:44:26.816Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:44:56.599Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:45:14.584Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:45:15.586Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:45:18.587Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:45:23.583Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:46:23.586Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:46:30.345Z level=INFO source=main.go:1500 msg="Loading configuration file" filename=/etc/prometheus/config_out/prometheus.env.yaml
time=2026-05-07T15:46:30.362Z level=INFO source=kubernetes.go:312 msg="Using pod service account via in-cluster config" component="discovery manager scrape" discovery=kubernetes config=serviceMonitor/openshift-pipelines/openshift-results-api-monitor/0
time=2026-05-07T15:46:30.367Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:46:30.369Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:46:30.426Z level=INFO source=main.go:1540 msg="Completed loading of configuration file" db_storage=1.702µs remote_storage=1.814µs web_handler=389ns query_engine=1.366µs scrape=1.929413ms scrape_sd=1.196529ms notify=386.989µs notify_sd=11.955µs rules=63.220484ms tracing=5.98µs filename=/etc/prometheus/config_out/prometheus.env.yaml totalDuration=81.121339ms
time=2026-05-07T15:47:41.594Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:47:47.594Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:48:05.601Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:48:44.598Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:48:47.587Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:49:12.585Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:49:30.345Z level=INFO source=main.go:1500 msg="Loading configuration file" filename=/etc/prometheus/config_out/prometheus.env.yaml
time=2026-05-07T15:49:30.456Z level=INFO source=main.go:1540 msg="Completed loading of configuration file" db_storage=1.214µs remote_storage=1.788µs web_handler=393ns query_engine=949ns scrape=1.942542ms scrape_sd=802.35µs notify=343.159µs notify_sd=9.774µs rules=93.836954ms tracing=6.677µs filename=/etc/prometheus/config_out/prometheus.env.yaml totalDuration=110.741877ms
time=2026-05-07T15:49:56.596Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:49:59.606Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:50:08.604Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:50:11.588Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:50:15.598Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:50:28.595Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:50:31.600Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime
time=2026-05-07T15:50:36.597Z level=INFO source=warnings.go:70 msg="v1 Endpoints is deprecated in v1.33+; use discovery.k8s.io/v1 EndpointSlice" component=k8s_client_runtime