{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"setup","msg":"Cluster config","Operator Namespace":"opendatahub","Application Namespace":"opendatahub","Release":{"name":"Open Data Hub","version":"3.6.0-ea.2"},"Cluster":{"type":"OpenShift","version":"4.22.20"}}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/convert"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.builder","msg":"Conversion webhook enabled","GVK":"datasciencecluster.opendatahub.io/v1, Kind=DataScienceCluster"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-datasciencecluster-v1"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/mutate-datasciencecluster-v1"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-datasciencecluster-v2"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/mutate-datasciencecluster-v2"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.builder","msg":"Conversion webhook enabled","GVK":"dscinitialization.opendatahub.io/v1, Kind=DSCInitialization"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-dscinitialization-v1"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-dscinitialization-v2"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/mutate-hardware-profile"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/mutate-prometheus-monitors"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-dashboard-acceleratorprofile"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Registering webhook","path":"/validate-dashboard-hardwareprofile"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"setup","msg":"creating reconciler","type":"service","name":"auth"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"setup","msg":"creating reconciler","type":"service","name":"certconfigmapgenerator"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"setup","msg":"creating reconciler","type":"service","name":"gateway"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"setup","msg":"creating reconciler","type":"service","name":"setupcontroller"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"setup","msg":"creating reconciler","type":"component","name":"kueue"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"setup","msg":"starting manager"}
{"level":"info","ts":"2026-10-30T17:17:23Z","msg":"starting server","name":"health probe","addr":"[::]:8081"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Starting webhook server"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.metrics","msg":"Starting metrics server"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.webhook","msg":"Serving webhook server","host":"","port":9443}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.certwatcher","msg":"Updated current TLS certificate","cert":"/etc/metrics-certs/tls.crt","key":"/etc/metrics-certs/tls.key"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.metrics","msg":"Serving metrics server","bindAddress":":8443","secure":true}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/tmp/k8s-webhook-server/serving-certs/tls.crt","key":"/tmp/k8s-webhook-server/serving-certs/tls.key","interval":10}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"controller-runtime.certwatcher","msg":"Starting certificate poll+watcher","cert":"/etc/metrics-certs/tls.crt","key":"/etc/metrics-certs/tls.key","interval":10}
{"level":"info","ts":"2026-10-30T17:17:23Z","msg":"Starting EventSource","controller":"tlssecurityprofilewatcher","source":"kind source: *v1.APIServer"}
{"level":"info","ts":"2026-10-30T17:17:23Z","logger":"setup","msg":"Attempting to acquire leader lease...","lock":"opendatahub/07ed84f7.opendatahub.io"}
{"level":"info","ts":"2026-10-30T17:17:23Z","msg":"Starting Controller","controller":"tlssecurityprofilewatcher"}
{"level":"info","ts":"2026-10-30T17:17:23Z","msg":"Starting workers","controller":"tlssecurityprofilewatcher","worker count":1}
{"level":"error","ts":"2026-10-30T18:22:50Z","logger":"setup","msg":"Error retrieving lease lock","lock":"opendatahub/07ed84f7.opendatahub.io","error":"Get \"https://172.31.0.1:443/apis/coordination.k8s.io/v1/namespaces/opendatahub/leases/07ed84f7.opendatahub.io?timeout=5s\": context deadline exceeded","stacktrace":"k8s.io/client-go/tools/leaderelection.(*LeaderElector).tryAcquireOrRenew\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:461\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).acquire.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:261\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:223\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).acquire\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:259\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).Run\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:215\nsigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start.func3\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/manager/internal.go:470"}
{"level":"error","ts":"2026-10-30T18:22:58Z","msg":"Failed to get namespace","object":{"name":"maas-api-metrics","namespace":"odh-ai-gateway-infra"},"namespace":"odh-ai-gateway-infra","name":"maas-api-metrics","resource":{"group":"monitoring.coreos.com","version":"v1","resource":"servicemonitors"},"user":"system:serviceaccount:opendatahub:maas-controller","requestID":"42289ad9-bce5-4d09-a0d9-8d6b63191d46","webhook":"monitoring-injector","namespace":"odh-ai-gateway-infra","name":"maas-api-metrics","operation":"UPDATE","kind":"ServiceMonitor","namespace":"odh-ai-gateway-infra","error":"Get \"https://172.31.0.1:443/api/v1/namespaces/odh-ai-gateway-infra\": context canceled","stacktrace":"github.com/opendatahub-io/opendatahub-operator/v2/internal/webhook/monitoring.(*Injector).performMonitoringInjection\n\t/workspace/internal/webhook/monitoring/mutating.go:170\ngithub.com/opendatahub-io/opendatahub-operator/v2/internal/webhook/monitoring.(*Injector).Handle\n\t/workspace/internal/webhook/monitoring/mutating.go:119\nsigs.k8s.io/controller-runtime/pkg/webhook/admission.(*Webhook).Handle\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/webhook/admission/webhook.go:181\nsigs.k8s.io/controller-runtime/pkg/webhook/admission.(*Webhook).ServeHTTP\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/webhook/admission/http.go:119\nsigs.k8s.io/controller-runtime/pkg/webhook/internal/metrics.InstrumentedHook.InstrumentHandlerInFlight.func1\n\t/opt/app-root/src/go/pkg/mod/github.com/prometheus/client_golang@v1.23.2/prometheus/promhttp/instrument_server.go:60\nnet/http.HandlerFunc.ServeHTTP\n\t/usr/lib/golang/src/net/http/server.go:2286\ngithub.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1\n\t/opt/app-root/src/go/pkg/mod/github.com/prometheus/client_golang@v1.23.2/prometheus/promhttp/instrument_server.go:147\nnet/http.HandlerFunc.ServeHTTP\n\t/usr/lib/golang/src/net/http/server.go:2286\ngithub.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2\n\t/opt/app-root/src/go/pkg/mod/github.com/prometheus/client_golang@v1.23.2/prometheus/promhttp/instrument_server.go:109\nnet/http.HandlerFunc.ServeHTTP\n\t/usr/lib/golang/src/net/http/server.go:2286\nnet/http.(*ServeMux).ServeHTTP\n\t/usr/lib/golang/src/net/http/server.go:2828\nnet/http.serverHandler.ServeHTTP\n\t/usr/lib/golang/src/net/http/server.go:3311\nnet/http.(*conn).serve\n\t/usr/lib/golang/src/net/http/server.go:2067"}
{"level":"error","ts":"2026-10-30T18:22:59Z","logger":"setup","msg":"Error retrieving lease lock","lock":"opendatahub/07ed84f7.opendatahub.io","error":"Get \"https://172.31.0.1:443/apis/coordination.k8s.io/v1/namespaces/opendatahub/leases/07ed84f7.opendatahub.io?timeout=5s\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"k8s.io/client-go/tools/leaderelection.(*LeaderElector).tryAcquireOrRenew\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:461\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).acquire.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:261\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:223\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).acquire\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:259\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).Run\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:215\nsigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start.func3\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/manager/internal.go:470"}
{"level":"error","ts":"2026-10-30T18:23:08Z","logger":"setup","msg":"Error retrieving lease lock","lock":"opendatahub/07ed84f7.opendatahub.io","error":"Get \"https://172.31.0.1:443/apis/coordination.k8s.io/v1/namespaces/opendatahub/leases/07ed84f7.opendatahub.io?timeout=5s\": context deadline exceeded","stacktrace":"k8s.io/client-go/tools/leaderelection.(*LeaderElector).tryAcquireOrRenew\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:461\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).acquire.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:261\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:223\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).acquire\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:259\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).Run\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:215\nsigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start.func3\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/manager/internal.go:470"}
{"level":"error","ts":"2026-10-30T18:23:15Z","logger":"setup","msg":"Error retrieving lease lock","lock":"opendatahub/07ed84f7.opendatahub.io","error":"Get \"https://172.31.0.1:443/apis/coordination.k8s.io/v1/namespaces/opendatahub/leases/07ed84f7.opendatahub.io?timeout=5s\": context deadline exceeded","stacktrace":"k8s.io/client-go/tools/leaderelection.(*LeaderElector).tryAcquireOrRenew\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:461\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).acquire.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:261\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:223\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).acquire\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:259\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).Run\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:215\nsigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start.func3\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/manager/internal.go:470"}
{"level":"error","ts":"2026-10-30T18:23:23Z","logger":"setup","msg":"Error retrieving lease lock","lock":"opendatahub/07ed84f7.opendatahub.io","error":"Get \"https://172.31.0.1:443/apis/coordination.k8s.io/v1/namespaces/opendatahub/leases/07ed84f7.opendatahub.io?timeout=5s\": context deadline exceeded (Client.Timeout exceeded while awaiting headers)","stacktrace":"k8s.io/client-go/tools/leaderelection.(*LeaderElector).tryAcquireOrRenew\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:461\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).acquire.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:261\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.JitterUntilWithContext\n\t/opt/app-root/src/go/pkg/mod/k8s.io/apimachinery@v0.36.2/pkg/util/wait/backoff.go:223\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).acquire\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:259\nk8s.io/client-go/tools/leaderelection.(*LeaderElector).Run\n\t/opt/app-root/src/go/pkg/mod/k8s.io/client-go@v0.36.1/tools/leaderelection/leaderelection.go:215\nsigs.k8s.io/controller-runtime/pkg/manager.(*controllerManager).Start.func3\n\t/opt/app-root/src/go/pkg/mod/sigs.k8s.io/controller-runtime@v0.24.1/pkg/manager/internal.go:470"}
{"level":"info","ts":"2026-10-30T18:23:27Z","logger":"controller-runtime.cache","msg":"Warning: watch ended with error","reflector":"go/pkg/mod/k8s.io/client-go@v0.36.1/tools/cache/reflector.go:343","type":"*v1.APIServer","err":"an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding"}
{"level":"info","ts":"2026-10-30T18:23:27Z","logger":"controller-runtime.cache","msg":"Warning: watch ended with error","reflector":"go/pkg/mod/k8s.io/client-go@v0.36.1/tools/cache/reflector.go:343","type":"config.openshift.io/v1, Kind=APIServer","err":"an error on the server (\"unable to decode an event from the watch stream: http2: client connection lost\") has prevented the request from succeeding"}