{"level":"info","ts":"2026-04-24T22:36:59.139708244Z","logger":"controller-runtime.builder","msg":"Registering a mutating webhook","GVK":"tempo.grafana.com/v1alpha1, Kind=TempoStack","path":"/mutate-tempo-grafana-com-v1alpha1-tempostack"}
{"level":"info","ts":"2026-04-24T22:36:59.139807383Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/mutate-tempo-grafana-com-v1alpha1-tempostack"}
{"level":"info","ts":"2026-04-24T22:36:59.139829731Z","logger":"controller-runtime.builder","msg":"Registering a validating webhook","GVK":"tempo.grafana.com/v1alpha1, Kind=TempoStack","path":"/validate-tempo-grafana-com-v1alpha1-tempostack"}
{"level":"info","ts":"2026-04-24T22:36:59.139853846Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-tempo-grafana-com-v1alpha1-tempostack"}
{"level":"info","ts":"2026-04-24T22:36:59.139892287Z","logger":"controller-runtime.builder","msg":"Registering a validating webhook","GVK":"tempo.grafana.com/v1alpha1, Kind=TempoMonolithic","path":"/validate-tempo-grafana-com-v1alpha1-tempomonolithic"}
{"level":"info","ts":"2026-04-24T22:36:59.139916211Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-tempo-grafana-com-v1alpha1-tempomonolithic"}
{"level":"info","ts":"2026-04-24T22:36:59.139939315Z","logger":"setup","msg":"Starting Tempo Operator","build-date":"2026-04-17T14:19:39Z","revision":"8896255a0bb96be44d7a56e0e8da00ed2602470d","tempo-operator":"0.20.0","tempo":"2.10.0","tempo-query":"2.10.0","default-tempo-image":"registry.redhat.io/rhosdt/tempo-rhel9@sha256:91b5138e11f428e1f8c77aa9651297084701cdbe799b8553235d70aa5b0afb83","default-tempo-query-image":"registry.redhat.io/rhosdt/tempo-query-rhel9@sha256:dd811d521e8334c831175d7a5e1202ed0cedaaa3ed09612280ebbe5f00178ba1","default-tempo-gateway-image":"registry.redhat.io/rhosdt/tempo-gateway-rhel9@sha256:5d5591d8c0639f2fe65eaba1d96c7127cfa538739ff6043ec408fb92a42537cb","default-tempo-gateway-opa-image":"registry.redhat.io/rhosdt/tempo-gateway-opa-rhel9@sha256:c5bacd8f492e4b2482c5fb52b1b64ef1fcc6a3db43185e2a68ead7bb8f50f917","default-network-policies":true,"go-version":"go1.25.8 (Red Hat 1.25.8-1.el9_7) X:strictfipsruntime","go-arch":"amd64","go-os":"linux"}
{"level":"info","ts":"2026-04-24T22:36:59.140145822Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"}
{"level":"info","ts":"2026-04-24T22:36:59.140200357Z","msg":"starting server","name":"health probe","addr":"[::]:8081"}
{"level":"info","ts":"2026-04-24T22:36:59.140235775Z","logger":"controller-runtime.webhook","msg":"Starting webhook server"}
{"level":"info","ts":"2026-04-24T22:36:59.140406488Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key"}
{"level":"info","ts":"2026-04-24T22:36:59.140423537Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/var/run/tls/server/tls.crt","key":"/var/run/tls/server/tls.key"}
{"level":"info","ts":"2026-04-24T22:36:59.140477459Z","logger":"controller-runtime.webhook","msg":"Serving webhook server","host":"","port":9443}
{"level":"info","ts":"2026-04-24T22:36:59.140499654Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":8443","secure":true}
{"level":"info","ts":"2026-04-24T22:36:59.140525535Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key","interval":10}
{"level":"info","ts":"2026-04-24T22:36:59.140573191Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/var/run/tls/server/tls.crt","key":"/var/run/tls/server/tls.key","interval":10}
{"level":"info","ts":"2026-04-24T22:36:59.241506893Z","msg":"attempting to acquire leader lease tempo-operator/8b886b0f.grafana.com..."}
{"level":"info","ts":"2026-04-24T22:36:59.262073989Z","msg":"successfully acquired lease tempo-operator/8b886b0f.grafana.com"}
{"level":"info","ts":"2026-04-24T22:36:59.262380587Z","msg":"Starting EventSource","controller":"certrotation_monolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.Secret"}
{"level":"info","ts":"2026-04-24T22:36:59.262782469Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1alpha1.TempoMonolithic"}
{"level":"info","ts":"2026-04-24T22:36:59.262803299Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-04-24T22:36:59.262840966Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.Secret"}
{"level":"info","ts":"2026-04-24T22:36:59.262861673Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.Route"}
{"level":"info","ts":"2026-04-24T22:36:59.262905773Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.RoleBinding"}
{"level":"info","ts":"2026-04-24T22:36:59.262923415Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.Secret"}
{"level":"info","ts":"2026-04-24T22:36:59.262911666Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.ServiceMonitor"}
{"level":"info","ts":"2026-04-24T22:36:59.262938779Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1alpha1.TempoStack"}
{"level":"info","ts":"2026-04-24T22:36:59.262962053Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.ConfigMap"}
{"level":"info","ts":"2026-04-24T22:36:59.262997074Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.ServiceAccount"}
{"level":"info","ts":"2026-04-24T22:36:59.26302549Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.Service"}
{"level":"info","ts":"2026-04-24T22:36:59.263038685Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.Secret"}
{"level":"info","ts":"2026-04-24T22:36:59.263030415Z","msg":"Starting EventSource","controller":"certrotation","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.Secret"}
{"level":"info","ts":"2026-04-24T22:36:59.263053613Z","msg":"Starting EventSource","controller":"certrotation_monolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1alpha1.TempoMonolithic"}
{"level":"info","ts":"2026-04-24T22:36:59.263060004Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.ServiceAccount"}
{"level":"info","ts":"2026-04-24T22:36:59.263071508Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.Role"}
{"level":"info","ts":"2026-04-24T22:36:59.263072253Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.StatefulSet"}
{"level":"info","ts":"2026-04-24T22:36:59.26309055Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.Ingress"}
{"level":"info","ts":"2026-04-24T22:36:59.263093315Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.Deployment"}
{"level":"info","ts":"2026-04-24T22:36:59.263083084Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.Role"}
{"level":"info","ts":"2026-04-24T22:36:59.263112914Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.ClusterRole"}
{"level":"info","ts":"2026-04-24T22:36:59.263123186Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.RoleBinding"}
{"level":"info","ts":"2026-04-24T22:36:59.263145048Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.Route"}
{"level":"info","ts":"2026-04-24T22:36:59.263143807Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.StatefulSet"}
{"level":"info","ts":"2026-04-24T22:36:59.263155264Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.ClusterRoleBinding"}
{"level":"info","ts":"2026-04-24T22:36:59.263202113Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.ServiceMonitor"}
{"level":"info","ts":"2026-04-24T22:36:59.263216148Z","msg":"Starting EventSource","controller":"certrotation","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1alpha1.TempoStack"}
{"level":"info","ts":"2026-04-24T22:36:59.263216759Z","msg":"Starting EventSource","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","source":"kind source: *v1.PrometheusRule"}
{"level":"info","ts":"2026-04-24T22:36:59.263314973Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.PrometheusRule"}
{"level":"info","ts":"2026-04-24T22:36:59.263339017Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.ClusterRole"}
{"level":"info","ts":"2026-04-24T22:36:59.26334601Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.Ingress"}
{"level":"info","ts":"2026-04-24T22:36:59.263355459Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.ConfigMap"}
{"level":"info","ts":"2026-04-24T22:36:59.263350182Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.Secret"}
{"level":"info","ts":"2026-04-24T22:36:59.263363917Z","msg":"Starting EventSource","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","source":"kind source: *v1.ClusterRoleBinding"}
{"level":"info","ts":"2026-04-24T22:36:59.478424019Z","msg":"discovered Kubernetes API server endpoints","ports":1,"ips":1}
{"level":"info","ts":"2026-04-24T22:36:59.565434781Z","msg":"Starting Controller","controller":"certrotation","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack"}
{"level":"info","ts":"2026-04-24T22:36:59.565457727Z","msg":"Starting workers","controller":"certrotation","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","worker count":1}
{"level":"info","ts":"2026-04-24T22:36:59.566679877Z","msg":"Starting Controller","controller":"certrotation_monolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic"}
{"level":"info","ts":"2026-04-24T22:36:59.566693333Z","msg":"Starting workers","controller":"certrotation_monolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","worker count":1}
{"level":"info","ts":"2026-04-24T22:36:59.56863197Z","msg":"Starting Controller","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic"}
{"level":"info","ts":"2026-04-24T22:36:59.56864187Z","msg":"Starting workers","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","worker count":1}
{"level":"info","ts":"2026-04-24T22:36:59.568649799Z","msg":"Starting Controller","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack"}
{"level":"info","ts":"2026-04-24T22:36:59.568659063Z","msg":"Starting workers","controller":"tempostack","controllerGroup":"tempo.grafana.com","controllerKind":"TempoStack","worker count":1}
{"level":"info","ts":"2026-04-24T22:37:19.265701236Z","logger":"certrotation-reconcile","msg":"Skipping reconciliation for unmanaged TempoMonolithic resource","controller":"certrotation_monolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"ab99c15a-1ff5-4361-84a3-6101acb89082","tempo":{"name":"tempo","namespace":"tempo-operator"},"name":"tempo-operator/tempo"}
{"level":"info","ts":"2026-04-24T22:37:19.440651514Z","logger":"certrotation-reconcile","msg":"Skipping reconciliation for unmanaged TempoMonolithic resource","controller":"certrotation_monolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"bf727fe7-2bdd-49c4-84af-6797e79c43f4","tempo":{"name":"tempo","namespace":"tempo-operator"},"name":"tempo-operator/tempo"}
{"level":"info","ts":"2026-04-24T22:37:19.458806482Z","logger":"certrotation-reconcile","msg":"Skipping reconciliation for unmanaged TempoMonolithic resource","controller":"certrotation_monolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"aab04da0-55f7-40e1-969d-986ad7f0a7d0","tempo":{"name":"tempo","namespace":"tempo-operator"},"name":"tempo-operator/tempo"}
{"level":"info","ts":"2026-04-24T22:37:19.472500442Z","logger":"tempomonolithic-reconcile","msg":"TempoMonolithic instances without multi-tenancy provide no authentication or authorization on the ingest or query paths, and are not supported on OpenShift","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"1a18e8b7-552d-43a7-9eea-3a92386ed945"}
{"level":"info","ts":"2026-04-24T22:37:19.472599627Z","logger":"tempomonolithic-reconcile","msg":"certificate rotation completed successfully, hash annotations added","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"1a18e8b7-552d-43a7-9eea-3a92386ed945","monolithic":"tempo-operator/tempo","event":"createOrRotateCerts"}
{"level":"info","ts":"2026-04-24T22:37:19.47267136Z","logger":"certrotation-reconcile","msg":"Skipping reconciliation for unmanaged TempoMonolithic resource","controller":"certrotation_monolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"11031316-4d62-487e-8d01-96535ffd7661","tempo":{"name":"tempo","namespace":"tempo-operator"},"name":"tempo-operator/tempo"}
{"level":"error","ts":"2026-04-24T22:37:19.479936252Z","msg":"Reconciler error","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"1a18e8b7-552d-43a7-9eea-3a92386ed945","error":"Operation cannot be fulfilled on tempomonolithics.tempo.grafana.com \"tempo\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-04-24T22:37:19.495822014Z","logger":"tempomonolithic-reconcile","msg":"TempoMonolithic instances without multi-tenancy provide no authentication or authorization on the ingest or query paths, and are not supported on OpenShift","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"e7d42754-c246-414c-925b-55cd162681bd"}
{"level":"info","ts":"2026-04-24T22:37:19.495944494Z","logger":"certrotation-reconcile","msg":"Skipping reconciliation for unmanaged TempoMonolithic resource","controller":"certrotation_monolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"541dd7b9-cc97-4700-b336-9afd5b0ebb96","tempo":{"name":"tempo","namespace":"tempo-operator"},"name":"tempo-operator/tempo"}
{"level":"info","ts":"2026-04-24T22:37:19.495967389Z","logger":"tempomonolithic-reconcile","msg":"certificate rotation completed successfully, hash annotations added","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"e7d42754-c246-414c-925b-55cd162681bd","monolithic":"tempo-operator/tempo","event":"createOrRotateCerts"}
{"level":"error","ts":"2026-04-24T22:37:19.50183403Z","msg":"Reconciler error","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"e7d42754-c246-414c-925b-55cd162681bd","error":"Operation cannot be fulfilled on tempomonolithics.tempo.grafana.com \"tempo\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\tsigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\tsigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\tsigs.k8s.io/controller-runtime@v0.22.4/pkg/internal/controller/controller.go:296"}
{"level":"info","ts":"2026-04-24T22:37:19.513530121Z","logger":"tempomonolithic-reconcile","msg":"TempoMonolithic instances without multi-tenancy provide no authentication or authorization on the ingest or query paths, and are not supported on OpenShift","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"ef72497f-126e-4b63-aaba-d71543b4c8f4"}
{"level":"info","ts":"2026-04-24T22:37:19.513674289Z","logger":"tempomonolithic-reconcile","msg":"certificate rotation completed successfully, hash annotations added","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"ef72497f-126e-4b63-aaba-d71543b4c8f4","monolithic":"tempo-operator/tempo","event":"createOrRotateCerts"}
{"level":"info","ts":"2026-04-24T22:37:19.531421466Z","logger":"tempomonolithic-reconcile","msg":"TempoMonolithic instances without multi-tenancy provide no authentication or authorization on the ingest or query paths, and are not supported on OpenShift","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"ef72497f-126e-4b63-aaba-d71543b4c8f4"}
{"level":"info","ts":"2026-04-24T22:37:19.531653199Z","logger":"certrotation-reconcile","msg":"Skipping reconciliation for unmanaged TempoMonolithic resource","controller":"certrotation_monolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"800270f3-0a2d-4234-a4a7-a85a22a6af1b","tempo":{"name":"tempo","namespace":"tempo-operator"},"name":"tempo-operator/tempo"}
{"level":"info","ts":"2026-04-24T22:37:19.672861254Z","logger":"certrotation-reconcile","msg":"Skipping reconciliation for unmanaged TempoMonolithic resource","controller":"certrotation_monolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"3f6e7106-d769-4a1b-b71f-569aed7136ed","tempo":{"name":"tempo","namespace":"tempo-operator"},"name":"tempo-operator/tempo"}
{"level":"info","ts":"2026-04-24T22:37:19.689122816Z","logger":"tempomonolithic-reconcile","msg":"TempoMonolithic instances without multi-tenancy provide no authentication or authorization on the ingest or query paths, and are not supported on OpenShift","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"e85c908e-f0a6-4135-b843-bdfa4aa83ba6"}
{"level":"info","ts":"2026-04-24T22:37:19.689256369Z","logger":"tempomonolithic-reconcile","msg":"certificate rotation completed successfully, hash annotations added","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"e85c908e-f0a6-4135-b843-bdfa4aa83ba6","monolithic":"tempo-operator/tempo","event":"createOrRotateCerts"}
{"level":"info","ts":"2026-04-24T22:37:19.78421684Z","logger":"tempomonolithic-reconcile","msg":"TempoMonolithic instances without multi-tenancy provide no authentication or authorization on the ingest or query paths, and are not supported on OpenShift","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"9c89ccd0-6543-4b55-b961-75d0e5938777"}
{"level":"info","ts":"2026-04-24T22:37:19.784434067Z","logger":"tempomonolithic-reconcile","msg":"certificate rotation completed successfully, hash annotations added","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"9c89ccd0-6543-4b55-b961-75d0e5938777","monolithic":"tempo-operator/tempo","event":"createOrRotateCerts"}
{"level":"info","ts":"2026-04-24T22:38:11.713343561Z","logger":"tempomonolithic-reconcile","msg":"TempoMonolithic instances without multi-tenancy provide no authentication or authorization on the ingest or query paths, and are not supported on OpenShift","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"f8d1a1ef-6f79-4b87-b62b-d0e26eaf1c3b"}
{"level":"info","ts":"2026-04-24T22:38:11.713485494Z","logger":"tempomonolithic-reconcile","msg":"certificate rotation completed successfully, hash annotations added","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"f8d1a1ef-6f79-4b87-b62b-d0e26eaf1c3b","monolithic":"tempo-operator/tempo","event":"createOrRotateCerts"}
{"level":"info","ts":"2026-04-24T22:38:11.866831486Z","logger":"certrotation-reconcile","msg":"Skipping reconciliation for unmanaged TempoMonolithic resource","controller":"certrotation_monolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"0850c0c2-1885-4a67-a999-677964a42d6c","tempo":{"name":"tempo","namespace":"tempo-operator"},"name":"tempo-operator/tempo"}
{"level":"info","ts":"2026-04-24T22:38:11.885056287Z","logger":"tempomonolithic-reconcile","msg":"TempoMonolithic instances without multi-tenancy provide no authentication or authorization on the ingest or query paths, and are not supported on OpenShift","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"b39a2fc5-3362-4c1e-baf4-bb8a00b78795"}
{"level":"info","ts":"2026-04-24T22:38:11.885182856Z","logger":"tempomonolithic-reconcile","msg":"certificate rotation completed successfully, hash annotations added","controller":"tempomonolithic","controllerGroup":"tempo.grafana.com","controllerKind":"TempoMonolithic","TempoMonolithic":{"name":"tempo","namespace":"tempo-operator"},"namespace":"tempo-operator","name":"tempo","reconcileID":"b39a2fc5-3362-4c1e-baf4-bb8a00b78795","monolithic":"tempo-operator/tempo","event":"createOrRotateCerts"}
{"level":"error","ts":"2026-04-24T22:53:30.96959368Z","msg":"Failed to update lock optimistically: Put \"https://172.30.0.1:443/apis/coordination.k8s.io/v1/namespaces/tempo-operator/leases/8b886b0f.grafana.com?timeout=5s\": context deadline exceeded, falling back to slow path","stacktrace":"go.uber.org/zap/zapio.(*Writer).log\n\tgo.uber.org/zap@v1.27.0/zapio/writer.go:146\ngo.uber.org/zap/zapio.(*Writer).writeLine\n\tgo.uber.org/zap@v1.27.0/zapio/writer.go:105\ngo.uber.org/zap/zapio.(*Writer).Write\n\tgo.uber.org/zap@v1.27.0/zapio/writer.go:83\nk8s.io/klog/v2.(*loggingT).output\n\tk8s.io/klog/v2@v2.130.1/klog.go:913\nk8s.io/klog/v2.(*loggingT).printfDepth\n\tk8s.io/klog/v2@v2.130.1/klog.go:760\nk8s.io/klog/v2.(*loggingT).printf\n\tk8s.io/klog/v2@v2.130.1/klog.go:737\nk8s.io/klog/v2.Errorf\n\tk8s.io/klog/v2@v2.130.1/klog.go:1597\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).tryAcquireOrRenew\n\tk8s.io/client-go@v0.34.3/tools/leaderelection/leaderelection.go:441\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).renew.func1.1\n\tk8s.io/client-go@v0.34.3/tools/leaderelection/leaderelection.go:285\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextTimeout\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/poll.go:48\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).renew.func1\n\tk8s.io/client-go@v0.34.3/tools/leaderelection/leaderelection.go:283\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:210\nk8s.io/apimachinery/pkg/util/wait.Until\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:163\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).renew\n\tk8s.io/client-go@v0.34.3/tools/leaderelection/leaderelection.go:282\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).Run\n\tk8s.io/client-go@v0.34.3/tools/leaderelection/leaderelection.go:221\nsigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start.func3\n\tsigs.k8s.io/controller-runtime@v0.22.4/pkg/manager/internal.go:454"}
{"level":"error","ts":"2026-04-24T22:53:35.970449332Z","msg":"error retrieving resource lock tempo-operator/8b886b0f.grafana.com: Get \"https://172.30.0.1:443/apis/coordination.k8s.io/v1/namespaces/tempo-operator/leases/8b886b0f.grafana.com?timeout=5s\": context deadline exceeded","stacktrace":"go.uber.org/zap/zapio.(*Writer).log\n\tgo.uber.org/zap@v1.27.0/zapio/writer.go:146\ngo.uber.org/zap/zapio.(*Writer).writeLine\n\tgo.uber.org/zap@v1.27.0/zapio/writer.go:105\ngo.uber.org/zap/zapio.(*Writer).Write\n\tgo.uber.org/zap@v1.27.0/zapio/writer.go:83\nk8s.io/klog/v2.(*loggingT).output\n\tk8s.io/klog/v2@v2.130.1/klog.go:913\nk8s.io/klog/v2.(*loggingT).printfDepth\n\tk8s.io/klog/v2@v2.130.1/klog.go:760\nk8s.io/klog/v2.(*loggingT).printf\n\tk8s.io/klog/v2@v2.130.1/klog.go:737\nk8s.io/klog/v2.Errorf\n\tk8s.io/klog/v2@v2.130.1/klog.go:1597\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).tryAcquireOrRenew\n\tk8s.io/client-go@v0.34.3/tools/leaderelection/leaderelection.go:448\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).renew.func1.1\n\tk8s.io/client-go@v0.34.3/tools/leaderelection/leaderelection.go:285\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext.func1\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/loop.go:53\nk8s.io/apimachinery/pkg/util/wait.loopConditionUntilContext\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/loop.go:54\nk8s.io/apimachinery/pkg/util/wait.PollUntilContextTimeout\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/poll.go:48\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).renew.func1\n\tk8s.io/client-go@v0.34.3/tools/leaderelection/leaderelection.go:283\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.JitterUntil\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:210\nk8s.io/apimachinery/pkg/util/wait.Until\n\tk8s.io/apimachinery@v0.34.3/pkg/util/wait/backoff.go:163\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).renew\n\tk8s.io/client-go@v0.34.3/tools/leaderelection/leaderelection.go:282\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).Run\n\tk8s.io/client-go@v0.34.3/tools/leaderelection/leaderelection.go:221\nsigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start.func3\n\tsigs.k8s.io/controller-runtime@v0.22.4/pkg/manager/internal.go:454"}
{"level":"info","ts":"2026-04-24T22:53:35.970521656Z","msg":"failed to renew lease tempo-operator/8b886b0f.grafana.com: context deadline exceeded"}
{"level":"error","ts":"2026-04-24T22:53:35.970586502Z","logger":"setup","msg":"problem running manager","error":"leader election lost","stacktrace":"github.com/grafana/tempo-operator/cmd/start.start\n\tgithub.com/grafana/tempo-operator/cmd/start/main.go:144\ngithub.com/spf13/cobra.(*Command).execute\n\tgithub.com/spf13/cobra@v1.10.2/command.go:1019\ngithub.com/spf13/cobra.(*Command).ExecuteC\n\tgithub.com/spf13/cobra@v1.10.2/command.go:1148\ngithub.com/spf13/cobra.(*Command).Execute\n\tgithub.com/spf13/cobra@v1.10.2/command.go:1071\nmain.main\n\t./main.go:26\nruntime.main\n\truntime/proc.go:285"}
{"level":"info","ts":"2026-04-24T22:53:35.970617045Z","msg":"Stopping and waiting for warmup runnables"}