==================== ISSUES SUMMARY ==================== ========== ODH/RHOAI ISSUES SUMMARY (status, events, highlights only) ========== redhat-ods-applications/maas-controller: ready=0/1 available=0 173m Warning FailedMount pod/mlflow-operator-controller-manager-8646f94cf4-dsb84 MountVolume.SetUp failed for volume "metrics-cert" : secret "controller-manager-metrics-tls" not found 169m Warning Unhealthy pod/mlflow-668464596d-hz95l Readiness probe failed: Get "https://10.131.3.15:8443/mlflow/health": dial tcp 10.131.3.15:8443: connect: connection refused 166m Warning Unhealthy pod/mlflow-58f77fb56c-gsnbt Readiness probe failed: Get "https://10.131.3.18:8443/mlflow/health": dial tcp 10.131.3.18:8443: connect: connection refused 141m Warning FailedToRetrieveImagePullSecret pod/maas-controller-656b6867cb-579m7 Unable to retrieve some image pull secrets (maas-controller-dockercfg-dxqp7); attempting to pull the image may not succeed. 137m Warning InvalidConfiguration servicemonitor/odh-model-controller-metrics-monitor ServiceMonitor odh-model-controller-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 137m Warning InvalidConfiguration servicemonitor/controller-manager-metrics-monitor ServiceMonitor controller-manager-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 137m Warning FailedToUpdateEndpoint endpoints/data-science-pipelines-operator-service Failed to update endpoint redhat-ods-applications/data-science-pipelines-operator-service: Operation cannot be fulfilled on endpoints "data-science-pipelines-operator-service": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/data-science-pipelines-operator-service, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: db7d9fd9-12e1-4a82-afa4-d25e6e315f6d, UID in object meta: 130m Warning FailedMount pod/spark-pi-d92jirumdfuc7b8vqjcg-driver MountVolume.SetUp failed for volume "spark-conf-volume-driver" : configmap "spark-drv-b5d1cc9f1e688798-conf-map" not found 130m Warning FailedMount pod/spark-pi-d92jirumdfuc7b8vqjcg-driver MountVolume.SetUp failed for volume "pod-template-volume" : configmap "spark-pi-d92jirumdfuc7b8vqjcg-08a6e89f1e68760a-driver-podspec-conf-map" not found 130m Warning FailedMount pod/spark-pi-ea37389f1e68a9c1-exec-1 MountVolume.SetUp failed for volume "spark-conf-volume-exec" : configmap "spark-exec-1fae0e9f1e68b3be-conf-map" not found 129m Warning FailedMount pod/scheduled-spark-pi-d92jj8emdfuc7b8vqjd0-1782921663001003395-driver MountVolume.SetUp failed for volume "spark-conf-volume-driver" : configmap "spark-drv-6b18dc9f1e69bd02-conf-map" not found 129m Warning FailedMount pod/scheduled-spark-pi-d92jj8emdfuc7b8vqjd0-1782921663001003395-driver MountVolume.SetUp failed for volume "pod-template-volume" : configmap "scheduled-spark-pi-d92jj8emdfuc7b8vqjd0-1782921663001003395-221b7d9f1e69ac3b-driver-podspec-conf-map" not found 125m Warning FailedMount pod/kuberay-operator-6b449f854b-dbllj MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found 118m Warning FailedMount pod/ogx-k8s-operator-controller-manager-67866ffc79-nnfmb MountVolume.SetUp failed for volume "cert" : secret "ogx-k8s-operator-webhook-cert" not found 118m Warning FailedMount pod/model-registry-operator-controller-manager-5f68f4fd76-q7jvp MountVolume.SetUp failed for volume "cert" : secret "model-registry-operator-webhook-service" not found 118m Warning FailedMount pod/odh-model-controller-6b647f87c-lvnwt MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found 118m Warning FailedMount pod/data-science-pipelines-operator-controller-manager-7fc7f76zzj88 MountVolume.SetUp failed for volume "config" : configmap "data-science-pipelines-operator-dspo-config" not found 118m Warning FailedMount pod/odh-notebook-controller-manager-6646bb96b9-wfsjr MountVolume.SetUp failed for volume "webhook-cert" : secret "odh-notebook-controller-webhook-cert" not found 118m Warning FailedCreate replicaset/model-serving-api-78bbbb9bf8 Error creating: pods "model-serving-api-78bbbb9bf8-" is forbidden: error looking up service account redhat-ods-applications/model-serving-api: serviceaccount "model-serving-api" not found 118m Warning FailedCreate replicaset/notebook-controller-deployment-5d7d79cf5b Error creating: pods "notebook-controller-deployment-5d7d79cf5b-" is forbidden: error looking up service account redhat-ods-applications/notebook-controller-service-account: serviceaccount "notebook-controller-service-account" not found 118m Warning FailedCreate replicaset/rhods-dashboard-6b5cc9785c Error creating: pods "rhods-dashboard-6b5cc9785c-" is forbidden: error looking up service account redhat-ods-applications/rhods-dashboard: serviceaccount "rhods-dashboard" not found 118m Warning FailedMount pod/rhods-dashboard-6b5cc9785c-tmkqj MountVolume.SetUp failed for volume "modules-sa-token" : secret "odh-dashboard-modules-token" not found 118m Warning FailedMount pod/rhods-dashboard-6b5cc9785c-v84xf MountVolume.SetUp failed for volume "modules-sa-token" : secret "odh-dashboard-modules-token" not found 117m Warning FailedCreate replicaset/maas-api-84c4b8dcb6 Error creating: pods "maas-api-84c4b8dcb6-" is forbidden: error looking up service account redhat-ods-applications/maas-api: serviceaccount "maas-api" not found 117m Warning FailedMount pod/maas-api-84c4b8dcb6-dh4pf MountVolume.SetUp failed for volume "maas-api-tls" : secret "maas-api-serving-cert" not found 117m Warning Unhealthy pod/feast-operator-controller-manager-744655f689-8qffq Readiness probe failed: Get "http://10.131.1.222:8081/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers) 117m Warning CalculateExpectedPodCountFailed poddisruptionbudget/ogx-k8s-operator-controller-manager-pdb Failed to calculate the number of expected pods: found no controllers for pod "ogx-k8s-operator-controller-manager-67866ffc79-nnfmb" 117m Warning CalculateExpectedPodCountFailed poddisruptionbudget/ogx-k8s-operator-controller-manager-pdb Failed to calculate the number of expected pods: found no controllers for pod "ogx-k8s-operator-controller-manager-67866ffc79-g25nm" 116m Warning FailedCreate replicaset/spark-operator-controller-57c5867998 Error creating: pods "spark-operator-controller-57c5867998-" is forbidden: error looking up service account redhat-ods-applications/spark-operator-controller: serviceaccount "spark-operator-controller" not found 116m Warning Unhealthy pod/rhods-dashboard-6b5cc9785c-tmkqj Readiness probe failed: Get "https://10.131.1.223:8444/healthz": dial tcp 10.131.1.223:8444: connect: connection refused 115m Warning FailedCreate replicaset/kuberay-operator-6b449f854b Error creating: pods "kuberay-operator-6b449f854b-" is forbidden: error looking up service account redhat-ods-applications/kuberay-operator: serviceaccount "kuberay-operator" not found 115m Warning FailedMount pod/spark-pi-d92jq26mdfuc74ubedpg-driver MountVolume.SetUp failed for volume "spark-conf-volume-driver" : configmap "spark-drv-4400ec9f1e769858-conf-map" not found 115m Warning FailedMount pod/spark-pi-d92jq26mdfuc74ubedpg-driver MountVolume.SetUp failed for volume "pod-template-volume" : configmap "spark-pi-d92jq26mdfuc74ubedpg-a4319f9f1e76866e-driver-podspec-conf-map" not found 114m Warning FailedMount pod/scheduled-spark-pi-d92jqemmdfuc74ubedq0-1782922585000972660-driver MountVolume.SetUp failed for volume "spark-conf-volume-driver" : configmap "spark-drv-d6d8159f1e77d272-conf-map" not found 114m Warning FailedMount pod/spark-pi-4502dd9f1e77f3d3-exec-1 MountVolume.SetUp failed for volume "spark-conf-volume-exec" : configmap "spark-exec-2e98df9f1e77fd62-conf-map" not found 107m Warning FailedMount pod/model-registry-operator-controller-manager-5f68f4fd76-ql2hj MountVolume.SetUp failed for volume "cert" : secret "model-registry-operator-webhook-service" not found 107m Warning InvalidConfiguration servicemonitor/odh-model-controller-metrics-monitor ServiceMonitor odh-model-controller-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 107m Warning InvalidConfiguration servicemonitor/workload-variant-autoscaler-controller-manager-metrics-monitor ServiceMonitor workload-variant-autoscaler-controller-manager-metrics-monitor was rejected due to invalid configuration: failed to get authorization token of type "": unable to get secret "workload-variant-autoscaler-metrics-reader-token": secrets "workload-variant-autoscaler-metrics-reader-token" not found 99m Warning FailedMount pod/odh-model-controller-54bf759f64-6v8p2 MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found 99m Warning FailedCreate replicaset/model-serving-api-78bbbb9bf8 Error creating: pods "model-serving-api-78bbbb9bf8-" is forbidden: error looking up service account redhat-ods-applications/model-serving-api: serviceaccount "model-serving-api" not found 98m Warning FailedCreate replicaset/maas-api-84c4b8dcb6 Error creating: pods "maas-api-84c4b8dcb6-" is forbidden: error looking up service account redhat-ods-applications/maas-api: serviceaccount "maas-api" not found 98m Warning InvalidConfiguration servicemonitor/controller-manager-metrics-monitor ServiceMonitor controller-manager-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 98m Warning FailedCreate replicaset/trustyai-service-operator-controller-manager-6f7f88c874 Error creating: pods "trustyai-service-operator-controller-manager-6f7f88c874-" is forbidden: error looking up service account redhat-ods-applications/trustyai-service-operator-controller-manager: serviceaccount "trustyai-service-operator-controller-manager" not found 96m Warning InvalidConfiguration servicemonitor/odh-model-controller-metrics-monitor ServiceMonitor odh-model-controller-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 95m Warning FailedCreate replicaset/model-serving-api-78bbbb9bf8 Error creating: pods "model-serving-api-78bbbb9bf8-" is forbidden: error looking up service account redhat-ods-applications/model-serving-api: serviceaccount "model-serving-api" not found 94m Warning FailedCreate replicaset/maas-api-84c4b8dcb6 Error creating: pods "maas-api-84c4b8dcb6-" is forbidden: error looking up service account redhat-ods-applications/maas-api: serviceaccount "maas-api" not found 94m Warning InvalidConfiguration servicemonitor/odh-model-controller-metrics-monitor ServiceMonitor odh-model-controller-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 93m Warning BackOff pod/maas-controller-656b6867cb-x8bsw Back-off restarting failed container manager in pod maas-controller-656b6867cb-x8bsw_redhat-ods-applications(1add1903-0a83-4c80-a177-8c96185847c3) 87m Warning Unhealthy pod/odh-model-controller-54bf759f64-kf84x Readiness probe failed: Get "http://10.129.2.104:8081/readyz": dial tcp 10.129.2.104:8081: connect: connection refused 87m Warning Unhealthy pod/llmisvc-controller-manager-586c85c465-vfxhn Readiness probe failed: Get "http://10.129.2.108:8081/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers) 87m Warning FailedMount pod/odh-model-controller-54bf759f64-pcbk4 MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found 87m Warning InvalidConfiguration servicemonitor/odh-model-controller-metrics-monitor ServiceMonitor odh-model-controller-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 87m Warning FailedCreate replicaset/model-serving-api-78bbbb9bf8 Error creating: pods "model-serving-api-78bbbb9bf8-" is forbidden: error looking up service account redhat-ods-applications/model-serving-api: serviceaccount "model-serving-api" not found 86m Warning FailedMount pod/llmisvc-controller-manager-586c85c465-lrwll MountVolume.SetUp failed for volume "cert" : secret "llmisvc-webhook-server-cert" not found 85m Warning FailedMount pod/odh-model-controller-54bf759f64-hr25g MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found 85m Warning FailedCreate replicaset/model-serving-api-78bbbb9bf8 Error creating: pods "model-serving-api-78bbbb9bf8-" is forbidden: error looking up service account redhat-ods-applications/model-serving-api: serviceaccount "model-serving-api" not found 85m Warning FailedMount pod/llmisvc-controller-manager-586c85c465-64pgz MountVolume.SetUp failed for volume "cert" : secret "llmisvc-webhook-server-cert" not found 29m Warning FailedMount pod/maas-controller-656b6867cb-x8bsw MountVolume.SetUp failed for volume "kube-api-access-n8j9n" : failed to fetch token: serviceaccounts "maas-controller" not found 10m Warning FailedCreate replicaset/rhods-dashboard-6b5cc9785c Error creating: pods "rhods-dashboard-6b5cc9785c-" is forbidden: error looking up service account redhat-ods-applications/rhods-dashboard: serviceaccount "rhods-dashboard" not found 10m Warning FailedMount pod/rhods-dashboard-6b5cc9785c-4z7kp MountVolume.SetUp failed for volume "modules-sa-token" : references non-existent secret key: token 9m21s Warning FailedMount pod/kuberay-operator-6b449f854b-k2265 MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found 4m16s Warning FailedToRetrieveImagePullSecret pod/maas-controller-656b6867cb-x8bsw Unable to retrieve some image pull secrets (maas-controller-dockercfg-hz29q); attempting to pull the image may not succeed. 49s Warning InvalidConfiguration servicemonitor/odh-model-controller-metrics-monitor ServiceMonitor odh-model-controller-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 155m Warning Deleting modelregistry/e2e-test-registry Custom Resource e2e-test-registry is being deleted from the namespace rhoai-model-registries 155m Warning Deleting modelregistry/e2e-test-registry Custom Resource e2e-test-registry is being deleted from the namespace rhoai-model-registries 149m Warning Deleting modelregistry/test-model-registry-662874 Custom Resource test-model-registry-662874 is being deleted from the namespace rhoai-model-registries 154m Warning Deleting modelregistry/postgres-registry-1782920032573 Custom Resource postgres-registry-1782920032573 is being deleted from the namespace rhoai-model-registries 154m Warning Deleting modelregistry/postgres-registry-1782920032573 Custom Resource postgres-registry-1782920032573 is being deleted from the namespace rhoai-model-registries 153m Warning Deleting modelregistry/default-db-registry-1782920032573 Custom Resource default-db-registry-1782920032573 is being deleted from the namespace rhoai-model-registries 153m Warning Deleting modelregistry/default-db-registry-1782920032573 Custom Resource default-db-registry-1782920032573 is being deleted from the namespace rhoai-model-registries 156m Warning Unhealthy pod/mysql-db-registry-1782920032573-7db99586d5-fvk9n Readiness probe failed: mysql: [Warning] Using a password on the command line interface can be insecure.... 155m Warning FailedMount pod/e2e-test-registry-7bdd59c98-kpbb5 MountVolume.SetUp failed for volume "kube-rbac-proxy-config" : configmap "e2e-test-registry-kube-rbac-proxy-config" not found 153m Warning Unhealthy pod/default-db-registry-1782920032573-54468846cc-vnrx4 Startup probe failed: Get "http://10.131.3.42:8080/readyz/isDirty": context deadline exceeded (Client.Timeout exceeded while awaiting headers) 153m Warning Unhealthy pod/default-db-registry-1782920032573-54468846cc-vnrx4 Startup probe failed: HTTP probe failed with statuscode: 503 152m Warning Unhealthy pod/default-db-registry-1782920032573-postgres-7f54dd4c58-lnbsg Readiness probe errored: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1 152m Warning Unhealthy pod/model-registry-db-662874-7956bc68d9-zfb8w Readiness probe failed: mysql: [Warning] Using a password on the command line interface can be insecure.... 137m Warning Unhealthy pod/model-catalog-7cc8755dc9-5pb2t Readiness probe failed: Get "https://10.129.2.44:8888/healthz": dial tcp 10.129.2.44:8888: connect: connection refused 137m Warning Unhealthy pod/model-catalog-7cc8755dc9-5pb2t Readiness probe failed: dial tcp 10.129.2.44:8080: i/o timeout 118m Warning FailedMount pod/model-catalog-7cc8755dc9-7gplg MountVolume.SetUp failed for volume "kube-rbac-proxy-config" : configmap "model-catalog-kube-rbac-proxy-config" not found 117m Warning Unhealthy pod/model-catalog-7cc8755dc9-7gplg Readiness probe failed: dial tcp 10.130.2.161:8080: connect: connection refused 106m Warning FailedMount pod/model-catalog-7cc8755dc9-r49tg MountVolume.SetUp failed for volume "kube-rbac-proxy-tls" : secret "model-catalog-kube-rbac-proxy" not found 106m Warning FailedMount pod/model-catalog-7cc8755dc9-r49tg MountVolume.SetUp failed for volume "kube-rbac-proxy-config" : configmap "model-catalog-kube-rbac-proxy-config" not found 103m Warning Unhealthy pod/model-catalog-7cc8755dc9-r49tg Readiness probe failed: dial tcp 10.130.2.174:8080: connect: connection refused 103m Warning Unhealthy pod/model-catalog-7cc8755dc9-r49tg Readiness probe failed: Get "https://10.130.2.174:8888/healthz": dial tcp 10.130.2.174:8888: connect: connection refused {"level":"error","ts":"2026-07-01T18:04:50Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:05:50Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:06:37Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:07:24Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:07:56Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:08:47Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:09:40Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:10:10Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:10:40Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} --------------------------------------------------------------------------------------------------- ==================== STATUS REPORT ==================== ========== ODH/RHOAI STATUS REPORT (2026-07-01T18:10:45Z) ========== --- OpenShift / OCP version --- Client Version: 4.20.8 Kustomize Version: v5.6.0 Server Version: 4.19.18 Kubernetes Version: v1.32.9 --- Namespaces --- namespace/redhat-ods-applications namespace/redhat-ods-monitoring namespace/redhat-ods-operator namespace/rhoai-model-registries namespace/rhods-notebooks --- Subscriptions (redhat-ods-operator) --- NAME PACKAGE SOURCE CHANNEL rhods-operator rhods-operator rhoai-catalog-dev beta Subscription conditions (unhealthy): CatalogSourcesUnhealthy: status=False reason=AllCatalogSourcesHealthy message=all available catalogsources are healthy --- ClusterServiceVersions (redhat-ods-operator) --- NAME DISPLAY VERSION REPLACES PHASE authorino-operator.v1.4.0 Authorino Operator 1.4.0 authorino-operator.v1.3.1 Succeeded cert-manager-operator.v1.19.0 cert-manager Operator for Red Hat OpenShift 1.19.0 cert-manager-operator.v1.18.1 Succeeded custom-metrics-autoscaler.v2.19.0-1 Custom Metrics Autoscaler 2.19.0-1 custom-metrics-autoscaler.v2.18.1-2 Succeeded dns-operator.v1.4.0 DNS Operator 1.4.0 dns-operator.v1.3.1 Succeeded kueue-operator.v1.3.1 Red Hat build of Kueue 1.3.1 Succeeded limitador-operator.v1.4.0 Limitador Operator 1.4.0 limitador-operator.v1.3.1 Succeeded rhcl-operator.v1.4.0 Red Hat Connectivity Link 1.4.0 rhcl-operator.v1.3.4 Succeeded rhods-operator.3.5.0-ea.2 Red Hat OpenShift AI 3.5.0-ea.2 Succeeded servicemeshoperator3.v3.3.4 Red Hat OpenShift Service Mesh 3 3.3.4 servicemeshoperator3.v3.1.0 Succeeded --- CSVs not in Succeeded phase --- (all CSVs report phase Succeeded) --- InstallPlans (redhat-ods-operator, not Complete) --- (no non-Complete InstallPlans) --- DataScienceCluster --- NAME READY REASON default-dsc True DSC Ready summary: True reason= DSC component states: aipipelines: managementState=Removed dashboard: managementState=Managed feastoperator: managementState=Removed kserve: managementState=Managed kueue: managementState=Unmanaged llamastackoperator: managementState=n/a mlflowoperator: managementState=Removed modelregistry: managementState=Removed ogx: managementState=Removed ray: managementState=Managed sparkoperator: managementState=Removed trainer: managementState=Managed trainingoperator: managementState=Removed trustyai: managementState=Removed workbenches: managementState=Removed DSC conditions: Ready: status=True reason= message= ProvisioningSucceeded: status=True reason= message= ComponentsReady: status=True reason= message= AIPipelinesReady: status=False reason=Removed message=Component ManagementState is set to Removed DashboardReady: status=True reason= message= FeastOperatorReady: status=False reason=Removed message=Component ManagementState is set to Removed KserveLLMInferenceServiceDependencies: status=True reason= message= KserveLLMInferenceServiceWideEPDependencies: status=True reason= message= KserveReady: status=True reason= message= KueueReady: status=True reason= message= MLflowOperatorReady: status=False reason=Removed message=Component ManagementState is set to Removed ModelControllerReady: status=True reason= message= ModelRegistryReady: status=False reason=Removed message=Component ManagementState is set to Removed ModelsAsServiceReady: status=False reason=Removed message=Component ManagementState is set to Removed OGXReady: status=False reason=Removed message=Component ManagementState is set to Removed RayReady: status=True reason= message= SparkOperatorReady: status=False reason=Removed message=Component ManagementState is set to Removed TrainerReady: status=True reason= message= TrainingOperatorReady: status=False reason=Removed message=Component ManagementState is set to Removed TrustyAIReady: status=False reason=Removed message=Component ManagementState is set to Removed WorkbenchesReady: status=False reason=Removed message=Component ManagementState is set to Removed --- DSCInitialization --- NAME AGE PHASE CREATED AT default-dsci 2d4h Ready 2026-06-29T13:43:18Z DSCInitialization conditions: ReconcileComplete: status=True reason=ReconcileCompleted message=Reconcile completed successfully Available: status=True reason=ReconcileCompleted message=Reconcile completed successfully Progressing: status=False reason=ReconcileCompleted message=Reconcile completed successfully Degraded: status=False reason=ReconcileCompleted message=Reconcile completed successfully Upgradeable: status=True reason=ReconcileCompleted message=Reconcile completed successfully MonitoringReady: status=True reason=Ready message=Monitoring stack is initialized Ready: status=True reason= message= ProvisioningSucceeded: status=True reason= message= MonitoringStackAvailable: status=False reason=MetricsNotConfigured message=Metrics not configured in DSCI CR TempoAvailable: status=False reason=TracesNotConfigured message=Traces not configured in DSCI CR OpenTelemetryCollectorAvailable: status=False reason=MetricsNotConfiguredAndTracesNotConfigured message=Metrics not configured in DSCI CR Traces not configured in DSCI CR AlertingAvailable: status=False reason=AlertingNotConfigured message=Alerting not configured in DSCI CR ThanosQuerierAvailable: status=False reason=MetricsNotConfigured message=Metrics not configured in DSCI CR PersesAvailable: status=False reason=MetricsNotConfiguredAndTracesNotConfigured message=Perses requires at least Metrics or Traces to be configured NodeMetricsEndpointAvailable: status=False reason=MetricsNotConfigured message=Metrics not configured in DSCI CR --- Component CRs (Ready) --- NAME READY REASON URL dashboard.components.platform.opendatahub.io/default-dashboard True NAME READY REASON kserve.components.platform.opendatahub.io/default-kserve True NAME READY REASON kueue.components.platform.opendatahub.io/default-kueue True NAME READY REASON URI modelcontroller.components.platform.opendatahub.io/default-modelcontroller True NAME READY REASON ray.components.platform.opendatahub.io/default-ray True NAME READY REASON trainer.components.platform.opendatahub.io/default-trainer True --- Deployments not fully ready (ODH/RHOAI namespaces) --- redhat-ods-applications/maas-controller: ready=0/1 available=0 --- Pods not Running/Succeeded (ODH/RHOAI namespaces) --- (all pods Running or Succeeded) --------------------------------------------------------------------------------------------------- ==================== EVENTS ==================== ========== ODH/RHOAI EVENTS (Warning / Error / Failed) ========== --- namespace/redhat-ods-applications --- LAST SEEN TYPE REASON OBJECT MESSAGE 173m Warning FailedMount pod/mlflow-operator-controller-manager-8646f94cf4-dsb84 MountVolume.SetUp failed for volume "metrics-cert" : secret "controller-manager-metrics-tls" not found 169m Warning Unhealthy pod/mlflow-668464596d-hz95l Readiness probe failed: Get "https://10.131.3.15:8443/mlflow/health": dial tcp 10.131.3.15:8443: connect: connection refused 166m Warning Unhealthy pod/mlflow-58f77fb56c-gsnbt Readiness probe failed: Get "https://10.131.3.18:8443/mlflow/health": dial tcp 10.131.3.18:8443: connect: connection refused 141m Warning FailedToRetrieveImagePullSecret pod/maas-controller-656b6867cb-579m7 Unable to retrieve some image pull secrets (maas-controller-dockercfg-dxqp7); attempting to pull the image may not succeed. 137m Warning InvalidConfiguration servicemonitor/odh-model-controller-metrics-monitor ServiceMonitor odh-model-controller-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 137m Warning InvalidConfiguration servicemonitor/controller-manager-metrics-monitor ServiceMonitor controller-manager-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 137m Warning FailedToUpdateEndpoint endpoints/data-science-pipelines-operator-service Failed to update endpoint redhat-ods-applications/data-science-pipelines-operator-service: Operation cannot be fulfilled on endpoints "data-science-pipelines-operator-service": StorageError: invalid object, Code: 4, Key: /kubernetes.io/services/endpoints/redhat-ods-applications/data-science-pipelines-operator-service, ResourceVersion: 0, AdditionalErrorMsg: Precondition failed: UID in precondition: db7d9fd9-12e1-4a82-afa4-d25e6e315f6d, UID in object meta: 130m Warning FailedMount pod/spark-pi-d92jirumdfuc7b8vqjcg-driver MountVolume.SetUp failed for volume "spark-conf-volume-driver" : configmap "spark-drv-b5d1cc9f1e688798-conf-map" not found 130m Warning FailedMount pod/spark-pi-d92jirumdfuc7b8vqjcg-driver MountVolume.SetUp failed for volume "pod-template-volume" : configmap "spark-pi-d92jirumdfuc7b8vqjcg-08a6e89f1e68760a-driver-podspec-conf-map" not found 130m Warning FailedMount pod/spark-pi-ea37389f1e68a9c1-exec-1 MountVolume.SetUp failed for volume "spark-conf-volume-exec" : configmap "spark-exec-1fae0e9f1e68b3be-conf-map" not found 129m Warning FailedMount pod/scheduled-spark-pi-d92jj8emdfuc7b8vqjd0-1782921663001003395-driver MountVolume.SetUp failed for volume "spark-conf-volume-driver" : configmap "spark-drv-6b18dc9f1e69bd02-conf-map" not found 129m Warning FailedMount pod/scheduled-spark-pi-d92jj8emdfuc7b8vqjd0-1782921663001003395-driver MountVolume.SetUp failed for volume "pod-template-volume" : configmap "scheduled-spark-pi-d92jj8emdfuc7b8vqjd0-1782921663001003395-221b7d9f1e69ac3b-driver-podspec-conf-map" not found 125m Warning FailedMount pod/kuberay-operator-6b449f854b-dbllj MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found 118m Warning FailedMount pod/ogx-k8s-operator-controller-manager-67866ffc79-nnfmb MountVolume.SetUp failed for volume "cert" : secret "ogx-k8s-operator-webhook-cert" not found 118m Warning FailedMount pod/model-registry-operator-controller-manager-5f68f4fd76-q7jvp MountVolume.SetUp failed for volume "cert" : secret "model-registry-operator-webhook-service" not found 118m Warning FailedMount pod/odh-model-controller-6b647f87c-lvnwt MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found 118m Warning FailedMount pod/data-science-pipelines-operator-controller-manager-7fc7f76zzj88 MountVolume.SetUp failed for volume "config" : configmap "data-science-pipelines-operator-dspo-config" not found 118m Warning FailedMount pod/odh-notebook-controller-manager-6646bb96b9-wfsjr MountVolume.SetUp failed for volume "webhook-cert" : secret "odh-notebook-controller-webhook-cert" not found 118m Warning FailedCreate replicaset/model-serving-api-78bbbb9bf8 Error creating: pods "model-serving-api-78bbbb9bf8-" is forbidden: error looking up service account redhat-ods-applications/model-serving-api: serviceaccount "model-serving-api" not found 118m Warning FailedCreate replicaset/notebook-controller-deployment-5d7d79cf5b Error creating: pods "notebook-controller-deployment-5d7d79cf5b-" is forbidden: error looking up service account redhat-ods-applications/notebook-controller-service-account: serviceaccount "notebook-controller-service-account" not found 118m Warning FailedCreate replicaset/rhods-dashboard-6b5cc9785c Error creating: pods "rhods-dashboard-6b5cc9785c-" is forbidden: error looking up service account redhat-ods-applications/rhods-dashboard: serviceaccount "rhods-dashboard" not found 118m Warning FailedMount pod/rhods-dashboard-6b5cc9785c-tmkqj MountVolume.SetUp failed for volume "modules-sa-token" : secret "odh-dashboard-modules-token" not found 118m Warning FailedMount pod/rhods-dashboard-6b5cc9785c-v84xf MountVolume.SetUp failed for volume "modules-sa-token" : secret "odh-dashboard-modules-token" not found 117m Warning FailedCreate replicaset/maas-api-84c4b8dcb6 Error creating: pods "maas-api-84c4b8dcb6-" is forbidden: error looking up service account redhat-ods-applications/maas-api: serviceaccount "maas-api" not found 117m Warning FailedMount pod/maas-api-84c4b8dcb6-dh4pf MountVolume.SetUp failed for volume "maas-api-tls" : secret "maas-api-serving-cert" not found 117m Warning Unhealthy pod/feast-operator-controller-manager-744655f689-8qffq Readiness probe failed: Get "http://10.131.1.222:8081/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers) 117m Warning CalculateExpectedPodCountFailed poddisruptionbudget/ogx-k8s-operator-controller-manager-pdb Failed to calculate the number of expected pods: found no controllers for pod "ogx-k8s-operator-controller-manager-67866ffc79-nnfmb" 117m Warning CalculateExpectedPodCountFailed poddisruptionbudget/ogx-k8s-operator-controller-manager-pdb Failed to calculate the number of expected pods: found no controllers for pod "ogx-k8s-operator-controller-manager-67866ffc79-g25nm" 116m Warning FailedCreate replicaset/spark-operator-controller-57c5867998 Error creating: pods "spark-operator-controller-57c5867998-" is forbidden: error looking up service account redhat-ods-applications/spark-operator-controller: serviceaccount "spark-operator-controller" not found 116m Warning Unhealthy pod/rhods-dashboard-6b5cc9785c-tmkqj Readiness probe failed: Get "https://10.131.1.223:8444/healthz": dial tcp 10.131.1.223:8444: connect: connection refused 115m Warning FailedCreate replicaset/kuberay-operator-6b449f854b Error creating: pods "kuberay-operator-6b449f854b-" is forbidden: error looking up service account redhat-ods-applications/kuberay-operator: serviceaccount "kuberay-operator" not found 115m Warning FailedMount pod/spark-pi-d92jq26mdfuc74ubedpg-driver MountVolume.SetUp failed for volume "spark-conf-volume-driver" : configmap "spark-drv-4400ec9f1e769858-conf-map" not found 115m Warning FailedMount pod/spark-pi-d92jq26mdfuc74ubedpg-driver MountVolume.SetUp failed for volume "pod-template-volume" : configmap "spark-pi-d92jq26mdfuc74ubedpg-a4319f9f1e76866e-driver-podspec-conf-map" not found 114m Warning FailedMount pod/scheduled-spark-pi-d92jqemmdfuc74ubedq0-1782922585000972660-driver MountVolume.SetUp failed for volume "spark-conf-volume-driver" : configmap "spark-drv-d6d8159f1e77d272-conf-map" not found 114m Warning FailedMount pod/spark-pi-4502dd9f1e77f3d3-exec-1 MountVolume.SetUp failed for volume "spark-conf-volume-exec" : configmap "spark-exec-2e98df9f1e77fd62-conf-map" not found 107m Warning FailedMount pod/model-registry-operator-controller-manager-5f68f4fd76-ql2hj MountVolume.SetUp failed for volume "cert" : secret "model-registry-operator-webhook-service" not found 107m Warning InvalidConfiguration servicemonitor/odh-model-controller-metrics-monitor ServiceMonitor odh-model-controller-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 107m Warning InvalidConfiguration servicemonitor/workload-variant-autoscaler-controller-manager-metrics-monitor ServiceMonitor workload-variant-autoscaler-controller-manager-metrics-monitor was rejected due to invalid configuration: failed to get authorization token of type "": unable to get secret "workload-variant-autoscaler-metrics-reader-token": secrets "workload-variant-autoscaler-metrics-reader-token" not found 99m Warning FailedMount pod/odh-model-controller-54bf759f64-6v8p2 MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found 99m Warning FailedCreate replicaset/model-serving-api-78bbbb9bf8 Error creating: pods "model-serving-api-78bbbb9bf8-" is forbidden: error looking up service account redhat-ods-applications/model-serving-api: serviceaccount "model-serving-api" not found 98m Warning FailedCreate replicaset/maas-api-84c4b8dcb6 Error creating: pods "maas-api-84c4b8dcb6-" is forbidden: error looking up service account redhat-ods-applications/maas-api: serviceaccount "maas-api" not found 98m Warning InvalidConfiguration servicemonitor/controller-manager-metrics-monitor ServiceMonitor controller-manager-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 98m Warning FailedCreate replicaset/trustyai-service-operator-controller-manager-6f7f88c874 Error creating: pods "trustyai-service-operator-controller-manager-6f7f88c874-" is forbidden: error looking up service account redhat-ods-applications/trustyai-service-operator-controller-manager: serviceaccount "trustyai-service-operator-controller-manager" not found 96m Warning InvalidConfiguration servicemonitor/odh-model-controller-metrics-monitor ServiceMonitor odh-model-controller-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 95m Warning FailedCreate replicaset/model-serving-api-78bbbb9bf8 Error creating: pods "model-serving-api-78bbbb9bf8-" is forbidden: error looking up service account redhat-ods-applications/model-serving-api: serviceaccount "model-serving-api" not found 94m Warning FailedCreate replicaset/maas-api-84c4b8dcb6 Error creating: pods "maas-api-84c4b8dcb6-" is forbidden: error looking up service account redhat-ods-applications/maas-api: serviceaccount "maas-api" not found 94m Warning InvalidConfiguration servicemonitor/odh-model-controller-metrics-monitor ServiceMonitor odh-model-controller-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 93m Warning BackOff pod/maas-controller-656b6867cb-x8bsw Back-off restarting failed container manager in pod maas-controller-656b6867cb-x8bsw_redhat-ods-applications(1add1903-0a83-4c80-a177-8c96185847c3) 87m Warning Unhealthy pod/odh-model-controller-54bf759f64-kf84x Readiness probe failed: Get "http://10.129.2.104:8081/readyz": dial tcp 10.129.2.104:8081: connect: connection refused 87m Warning Unhealthy pod/llmisvc-controller-manager-586c85c465-vfxhn Readiness probe failed: Get "http://10.129.2.108:8081/readyz": context deadline exceeded (Client.Timeout exceeded while awaiting headers) 87m Warning FailedMount pod/odh-model-controller-54bf759f64-pcbk4 MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found 87m Warning InvalidConfiguration servicemonitor/odh-model-controller-metrics-monitor ServiceMonitor odh-model-controller-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits 87m Warning FailedCreate replicaset/model-serving-api-78bbbb9bf8 Error creating: pods "model-serving-api-78bbbb9bf8-" is forbidden: error looking up service account redhat-ods-applications/model-serving-api: serviceaccount "model-serving-api" not found 86m Warning FailedMount pod/llmisvc-controller-manager-586c85c465-lrwll MountVolume.SetUp failed for volume "cert" : secret "llmisvc-webhook-server-cert" not found 85m Warning FailedMount pod/odh-model-controller-54bf759f64-hr25g MountVolume.SetUp failed for volume "cert" : secret "odh-model-controller-webhook-cert" not found 85m Warning FailedCreate replicaset/model-serving-api-78bbbb9bf8 Error creating: pods "model-serving-api-78bbbb9bf8-" is forbidden: error looking up service account redhat-ods-applications/model-serving-api: serviceaccount "model-serving-api" not found 85m Warning FailedMount pod/llmisvc-controller-manager-586c85c465-64pgz MountVolume.SetUp failed for volume "cert" : secret "llmisvc-webhook-server-cert" not found 29m Warning FailedMount pod/maas-controller-656b6867cb-x8bsw MountVolume.SetUp failed for volume "kube-api-access-n8j9n" : failed to fetch token: serviceaccounts "maas-controller" not found 10m Warning FailedCreate replicaset/rhods-dashboard-6b5cc9785c Error creating: pods "rhods-dashboard-6b5cc9785c-" is forbidden: error looking up service account redhat-ods-applications/rhods-dashboard: serviceaccount "rhods-dashboard" not found 10m Warning FailedMount pod/rhods-dashboard-6b5cc9785c-4z7kp MountVolume.SetUp failed for volume "modules-sa-token" : references non-existent secret key: token 9m21s Warning FailedMount pod/kuberay-operator-6b449f854b-k2265 MountVolume.SetUp failed for volume "cert" : secret "kuberay-webhook-server-cert" not found 4m16s Warning FailedToRetrieveImagePullSecret pod/maas-controller-656b6867cb-x8bsw Unable to retrieve some image pull secrets (maas-controller-dockercfg-hz29q); attempting to pull the image may not succeed. 49s Warning InvalidConfiguration servicemonitor/odh-model-controller-metrics-monitor ServiceMonitor odh-model-controller-metrics-monitor was rejected due to invalid configuration: it accesses file system via bearer token file which Prometheus specification prohibits --- namespace/redhat-ods-monitoring --- (no events) --- namespace/redhat-ods-operator --- (no events) --- namespace/rhoai-model-registries --- LAST SEEN TYPE REASON OBJECT MESSAGE 155m Warning Deleting modelregistry/e2e-test-registry Custom Resource e2e-test-registry is being deleted from the namespace rhoai-model-registries 155m Warning Deleting modelregistry/e2e-test-registry Custom Resource e2e-test-registry is being deleted from the namespace rhoai-model-registries 149m Warning Deleting modelregistry/test-model-registry-662874 Custom Resource test-model-registry-662874 is being deleted from the namespace rhoai-model-registries 154m Warning Deleting modelregistry/postgres-registry-1782920032573 Custom Resource postgres-registry-1782920032573 is being deleted from the namespace rhoai-model-registries 154m Warning Deleting modelregistry/postgres-registry-1782920032573 Custom Resource postgres-registry-1782920032573 is being deleted from the namespace rhoai-model-registries 153m Warning Deleting modelregistry/default-db-registry-1782920032573 Custom Resource default-db-registry-1782920032573 is being deleted from the namespace rhoai-model-registries 153m Warning Deleting modelregistry/default-db-registry-1782920032573 Custom Resource default-db-registry-1782920032573 is being deleted from the namespace rhoai-model-registries 156m Warning Unhealthy pod/mysql-db-registry-1782920032573-7db99586d5-fvk9n Readiness probe failed: mysql: [Warning] Using a password on the command line interface can be insecure.... 155m Warning FailedMount pod/e2e-test-registry-7bdd59c98-kpbb5 MountVolume.SetUp failed for volume "kube-rbac-proxy-config" : configmap "e2e-test-registry-kube-rbac-proxy-config" not found 153m Warning Unhealthy pod/default-db-registry-1782920032573-54468846cc-vnrx4 Startup probe failed: Get "http://10.131.3.42:8080/readyz/isDirty": context deadline exceeded (Client.Timeout exceeded while awaiting headers) 153m Warning Unhealthy pod/default-db-registry-1782920032573-54468846cc-vnrx4 Startup probe failed: HTTP probe failed with statuscode: 503 152m Warning Unhealthy pod/default-db-registry-1782920032573-postgres-7f54dd4c58-lnbsg Readiness probe errored: rpc error: code = Unknown desc = command error: cannot register an exec PID: container is stopping, stdout: , stderr: , exit code -1 152m Warning Unhealthy pod/model-registry-db-662874-7956bc68d9-zfb8w Readiness probe failed: mysql: [Warning] Using a password on the command line interface can be insecure.... 137m Warning Unhealthy pod/model-catalog-7cc8755dc9-5pb2t Readiness probe failed: Get "https://10.129.2.44:8888/healthz": dial tcp 10.129.2.44:8888: connect: connection refused 137m Warning Unhealthy pod/model-catalog-7cc8755dc9-5pb2t Readiness probe failed: dial tcp 10.129.2.44:8080: i/o timeout 118m Warning FailedMount pod/model-catalog-7cc8755dc9-7gplg MountVolume.SetUp failed for volume "kube-rbac-proxy-config" : configmap "model-catalog-kube-rbac-proxy-config" not found 117m Warning Unhealthy pod/model-catalog-7cc8755dc9-7gplg Readiness probe failed: dial tcp 10.130.2.161:8080: connect: connection refused 106m Warning FailedMount pod/model-catalog-7cc8755dc9-r49tg MountVolume.SetUp failed for volume "kube-rbac-proxy-tls" : secret "model-catalog-kube-rbac-proxy" not found 106m Warning FailedMount pod/model-catalog-7cc8755dc9-r49tg MountVolume.SetUp failed for volume "kube-rbac-proxy-config" : configmap "model-catalog-kube-rbac-proxy-config" not found 103m Warning Unhealthy pod/model-catalog-7cc8755dc9-r49tg Readiness probe failed: dial tcp 10.130.2.174:8080: connect: connection refused 103m Warning Unhealthy pod/model-catalog-7cc8755dc9-r49tg Readiness probe failed: Get "https://10.130.2.174:8888/healthz": dial tcp 10.130.2.174:8888: connect: connection refused --- namespace/rhods-notebooks --- LAST SEEN TYPE REASON OBJECT MESSAGE --------------------------------------------------------------------------------------------------- ==================== OPERATOR LOG HIGHLIGHTS ==================== ========== ODH/RHOAI LOG HIGHLIGHTS (errors, webhooks, reconcile failures) ========== Source: rhods-operator pods in redhat-ods-operator (since-time=2026-07-01T18:04:20Z) --- redhat-ods-operator/rhods-operator-6f6bb89fdf-59gvk --- {"level":"error","ts":"2026-07-01T18:04:50Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:05:50Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:06:37Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:07:24Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:07:56Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:08:47Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:09:40Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:10:10Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:10:40Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} --------------------------------------------------------------------------------------------------- ==================== OPERATOR POD LOG: rhods-operator-6f6bb89fdf-59gvk.log ==================== {"level":"error","ts":"2026-07-01T18:04:50Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"error","ts":"2026-07-01T18:05:50Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects"} {"level":"info","ts":"2026-07-01T18:06:00Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects.createDashboardRedirects","msg":"Creating dashboard redirect resources","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d","dashboardRouteName":"rhods-dashboard","namespace":"redhat-ods-applications","currentSubdomain":"rh-ai"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:06:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c4a5bcec-e05c-4685-86d3-f3e95503f83d","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects"} {"level":"info","ts":"2026-07-01T18:06:15Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects.createDashboardRedirects","msg":"Creating dashboard redirect resources","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954","dashboardRouteName":"rhods-dashboard","namespace":"redhat-ods-applications","currentSubdomain":"rh-ai"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:06:15Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm","msg":"EnvoyFilter exposes internal implementation details that may change at any time. Prefer other APIs if possible, and exercise extreme caution, especially around upgrades.","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:06:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"71f4dfa0-ee8b-4ba2-aade-64514f831954","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"error","ts":"2026-07-01T18:06:37Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"reconcile","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"apply","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/monitoring.addMonitoringCapability"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/monitoring.deployMonitoringAdmissionPolicies"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/monitoring.deployMonitoringStackWithQuerierAndRestrictions"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/monitoring.deployTracingStack"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/monitoring.deployAlerting"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/monitoring.deployOpenTelemetryCollector"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/monitoring.deployPerses"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/monitoring.deployPersesTempoIntegration"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/monitoring.deployPersesPrometheusIntegration"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/monitoring.deployNodeMetricsEndpoint"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/monitoring.syncPrometheusWebTLSCA"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Executing action","controller":"monitoring","controllerGroup":"services.platform.opendatahub.io","controllerKind":"Monitoring","Monitoring":{"name":"default-monitoring"},"namespace":"","name":"default-monitoring","reconcileID":"7968b196-3f1e-4db2-bd21-364d0b0a14a5","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-07-01T18:06:59Z","logger":"DSCInitialization","msg":"Reconciling DSCInitialization.","controller":"dscinitialization","controllerGroup":"dscinitialization.opendatahub.io","controllerKind":"DSCInitialization","DSCInitialization":{"name":"default-dsci"},"namespace":"","name":"default-dsci","reconcileID":"b579d8e6-51e6-4b68-b1c4-29f39712fc29","DSCInitialization Request.Name":"default-dsci"} {"level":"info","ts":"2026-07-01T18:06:59Z","msg":"Application namespace set in DSCI not found, creating it with labels","controller":"dscinitialization","controllerGroup":"dscinitialization.opendatahub.io","controllerKind":"DSCInitialization","DSCInitialization":{"name":"default-dsci"},"namespace":"","name":"default-dsci","reconcileID":"b579d8e6-51e6-4b68-b1c4-29f39712fc29","name":"redhat-ods-applications"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects"} {"level":"info","ts":"2026-07-01T18:07:15Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects.createDashboardRedirects","msg":"Creating dashboard redirect resources","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01","dashboardRouteName":"rhods-dashboard","namespace":"redhat-ods-applications","currentSubdomain":"rh-ai"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:07:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0738a171-fa0c-46d0-bd5d-eee4aa5c5e01","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"error","ts":"2026-07-01T18:07:24Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects"} {"level":"info","ts":"2026-07-01T18:07:30Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects.createDashboardRedirects","msg":"Creating dashboard redirect resources","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140","dashboardRouteName":"rhods-dashboard","namespace":"redhat-ods-applications","currentSubdomain":"rh-ai"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:07:30Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"fad1cb06-9690-49f4-9d02-db545cf02140","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"error","ts":"2026-07-01T18:07:56Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects"} {"level":"info","ts":"2026-07-01T18:08:00Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects.createDashboardRedirects","msg":"Creating dashboard redirect resources","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800","dashboardRouteName":"rhods-dashboard","namespace":"redhat-ods-applications","currentSubdomain":"rh-ai"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:08:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3e6e452-acc2-4ab1-93de-08f473768800","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects"} {"level":"info","ts":"2026-07-01T18:08:15Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects.createDashboardRedirects","msg":"Creating dashboard redirect resources","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66","dashboardRouteName":"rhods-dashboard","namespace":"redhat-ods-applications","currentSubdomain":"rh-ai"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:08:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"0f845e8d-659b-4489-a8ac-8ee2d3e35f66","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects"} {"level":"info","ts":"2026-07-01T18:08:45Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects.createDashboardRedirects","msg":"Creating dashboard redirect resources","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390","dashboardRouteName":"rhods-dashboard","namespace":"redhat-ods-applications","currentSubdomain":"rh-ai"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:08:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"c3ee4b44-2546-4913-82a3-50bcbc366390","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-07-01T18:08:46Z","logger":"controller-runtime.cache","msg":"Warning: This version is deprecated. Use v1beta2 instead."} {"level":"error","ts":"2026-07-01T18:08:47Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects"} {"level":"info","ts":"2026-07-01T18:09:00Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects.createDashboardRedirects","msg":"Creating dashboard redirect resources","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e","dashboardRouteName":"rhods-dashboard","namespace":"redhat-ods-applications","currentSubdomain":"rh-ai"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:09:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"14ee4bd6-7a9c-4308-aed2-aa804cd1b59e","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects"} {"level":"info","ts":"2026-07-01T18:09:15Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects.createDashboardRedirects","msg":"Creating dashboard redirect resources","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592","dashboardRouteName":"rhods-dashboard","namespace":"redhat-ods-applications","currentSubdomain":"rh-ai"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:09:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"83aa911e-1cad-4ea4-9704-d4e3404e6592","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"error","ts":"2026-07-01T18:09:40Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects"} {"level":"info","ts":"2026-07-01T18:09:45Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects.createDashboardRedirects","msg":"Creating dashboard redirect resources","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7","dashboardRouteName":"rhods-dashboard","namespace":"redhat-ods-applications","currentSubdomain":"rh-ai"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:09:45Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"d401e0ef-8059-4cea-b345-a834b5fe64b7","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects"} {"level":"info","ts":"2026-07-01T18:10:00Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects.createDashboardRedirects","msg":"Creating dashboard redirect resources","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0","dashboardRouteName":"rhods-dashboard","namespace":"redhat-ods-applications","currentSubdomain":"rh-ai"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:00Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"91162aea-e70c-4fa9-ad50-05a2d3f675f0","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-07-01T18:10:10Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"test-ns-r4phu","reconcileID":"6e7b2097-f802-41ed-a0af-b1f410f8cc40"} {"level":"info","ts":"2026-07-01T18:10:10Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"test-ns-r4phu","reconcileID":"64cf19ab-c0cf-4c43-a5b3-36c3df72137b"} {"level":"info","ts":"2026-07-01T18:10:10Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"test-ns-r4phu","reconcileID":"b8052668-a7df-4748-b232-3b143ef002f5"} {"level":"info","ts":"2026-07-01T18:10:10Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"test-ns-taafe","reconcileID":"2ef75497-49cf-4c40-95a0-3b0fae710cd7"} {"level":"info","ts":"2026-07-01T18:10:10Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"test-ns-taafe","reconcileID":"e39b8eb0-417f-4510-92b3-c1fda2f54bb3"} {"level":"info","ts":"2026-07-01T18:10:10Z","msg":"Adding CA bundle configmap","controller":"cert-configmap-generator-controller","namespace":"","name":"test-ns-taafe","reconcileID":"db629bde-256c-441a-8812-50f0dce1a145"} {"level":"error","ts":"2026-07-01T18:10:10Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"reconcile","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"apply","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.checkPreConditions"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.initialize"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.manageDefaultKueueResourcesAction"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.manageKueueAdminRoleBinding"} {"level":"info","ts":"2026-07-01T18:10:13Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.manageKueueAdminRoleBinding","msg":"Successfully managed kueue admin role binding","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","adminGroups":["rhods-admins"]} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.configureClusterQueueViewerRoleAction"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"298df850-39d9-4d01-98a7-4698e0383d22","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6f4fc387-71d0-48e9-8cbb-a18f0c3f0da9"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6f4fc387-71d0-48e9-8cbb-a18f0c3f0da9"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6f4fc387-71d0-48e9-8cbb-a18f0c3f0da9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6f4fc387-71d0-48e9-8cbb-a18f0c3f0da9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6f4fc387-71d0-48e9-8cbb-a18f0c3f0da9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6f4fc387-71d0-48e9-8cbb-a18f0c3f0da9","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"reconcile","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6f4fc387-71d0-48e9-8cbb-a18f0c3f0da9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"apply","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.checkPreConditions"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.initialize"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/releases.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/kustomize.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.manageDefaultKueueResourcesAction"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6f4fc387-71d0-48e9-8cbb-a18f0c3f0da9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"6f4fc387-71d0-48e9-8cbb-a18f0c3f0da9","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.manageKueueAdminRoleBinding"} {"level":"info","ts":"2026-07-01T18:10:13Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.manageKueueAdminRoleBinding","msg":"Successfully managed kueue admin role binding","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","adminGroups":["rhods-admins"]} {"level":"info","ts":"2026-07-01T18:10:13Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/status/deployments.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.(*componentHandler).NewComponentReconciler.func1"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/components/kueue.configureClusterQueueViewerRoleAction"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"Executing action","controller":"kueue","controllerGroup":"components.platform.opendatahub.io","controllerKind":"Kueue","Kueue":{"name":"default-kueue"},"namespace":"","name":"default-kueue","reconcileID":"ce95eb33-8b7c-4ef6-ba9f-950d72ea4b5a","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"reconcile","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25fd5b4-0800-4178-9f13-76d9d6a72c66"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"apply","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25fd5b4-0800-4178-9f13-76d9d6a72c66"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25fd5b4-0800-4178-9f13-76d9d6a72c66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.initialize"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25fd5b4-0800-4178-9f13-76d9d6a72c66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.checkPreConditions"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25fd5b4-0800-4178-9f13-76d9d6a72c66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.updateStatus"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25fd5b4-0800-4178-9f13-76d9d6a72c66","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/datasciencecluster.provisionComponents"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25fd5b4-0800-4178-9f13-76d9d6a72c66","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25fd5b4-0800-4178-9f13-76d9d6a72c66","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:14Z","msg":"Executing action","controller":"datasciencecluster","controllerGroup":"datasciencecluster.opendatahub.io","controllerKind":"DataScienceCluster","DataScienceCluster":{"name":"default-dsc"},"namespace":"","name":"default-dsc","reconcileID":"d25fd5b4-0800-4178-9f13-76d9d6a72c66","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"reconcile","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"apply","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createGatewayInfrastructure"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createKubeAuthProxyInfrastructure"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createEnvoyFilter"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createNetworkPolicy"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createOCPRoutes"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects"} {"level":"info","ts":"2026-07-01T18:10:15Z","logger":"action.github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.createDashboardRedirects.createDashboardRedirects","msg":"Creating dashboard redirect resources","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd","dashboardRouteName":"rhods-dashboard","namespace":"redhat-ods-applications","currentSubdomain":"rh-ai"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/render/template.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/deploy.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd","action":"github.com/opendatahub-io/opendatahub-operator/v2/internal/controller/services/gateway.syncGatewayConfigStatus"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/actions/gc.(*Action).run-fm"} {"level":"info","ts":"2026-07-01T18:10:15Z","msg":"Executing action","controller":"gatewayconfig","controllerGroup":"services.platform.opendatahub.io","controllerKind":"GatewayConfig","GatewayConfig":{"name":"default-gateway"},"namespace":"","name":"default-gateway","reconcileID":"bcf1dd9a-9d84-48c3-9dce-f2a946f8c4bd","action":"github.com/opendatahub-io/opendatahub-operator/v2/pkg/controller/reconciler.(*dynamicWatchAction).run-fm"} {"level":"error","ts":"2026-07-01T18:10:40Z","logger":"controller-runtime.cache.UnhandledError","msg":"Failed to watch","reflector":"pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:289","type":"operator.authorino.kuadrant.io/v1beta1, Kind=Authorino","error":"authorinos.operator.authorino.kuadrant.io is forbidden: User \"system:serviceaccount:redhat-ods-operator:redhat-ods-operator-controller-manager\" cannot watch resource \"authorinos\" in API group \"operator.authorino.kuadrant.io\" at the cluster scope","stacktrace":"k8s.io/apimachinery/pkg/util/runtime.logError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:221\nk8s.io/apimachinery/pkg/util/runtime.handleError\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:212\nk8s.io/apimachinery/pkg/util/runtime.HandleErrorWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/runtime/runtime.go:198\nk8s.io/client-go/tools/cache.DefaultWatchErrorHandler\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:204\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:370\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:255\nk8s.io/apimachinery/pkg/util/wait.BackoffUntilWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:256\nk8s.io/apimachinery/pkg/util/wait.BackoffUntil\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/backoff.go:233\nk8s.io/client-go/tools/cache.(*Reflector).RunWithContext\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/client-go@v0.35.3/tools/cache/reflector.go:368\nk8s.io/client-go/tools/cache.(*controller).RunWithContext.(*Group).StartWithContext.func3\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:63\nk8s.io/apimachinery/pkg/util/wait.(*Group).Start.func1\n\t/cachi2/output/deps/gomod/pkg/mod/k8s.io/apimachinery@v0.35.3/pkg/util/wait/wait.go:72"} ==================== OPERATOR POD LOG: rhods-operator-6f6bb89fdf-f4zpw.log ==================== ==================== OPERATOR POD LOG: rhods-operator-6f6bb89fdf-kr2pd.log ==================== ==================== WORKLOAD POD LOG: redhat-ods-applications/dashboard-redirect-6bd46884d4-7zbzz.log ==================== ==================== WORKLOAD POD LOG: redhat-ods-applications/kserve-controller-manager-6c8cb86d7f-t72g9.log ==================== ==================== WORKLOAD POD LOG: redhat-ods-applications/kubeflow-trainer-controller-manager-776d978f97-855zk.log ==================== ==================== WORKLOAD POD LOG: redhat-ods-applications/kuberay-operator-6b449f854b-k2265.log ==================== {"level":"info","ts":"2026-07-01T18:10:11.001Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"fecc2fa4-bd98-4c75-8ef6-26edf27e0bdd","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"} {"level":"info","ts":"2026-07-01T18:10:11.001Z","logger":"controllers.RayJob","msg":"Add a finalizer","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"fecc2fa4-bd98-4c75-8ef6-26edf27e0bdd","finalizer":"ray.io/rayjob-finalizer"} {"level":"info","ts":"2026-07-01T18:10:11.022Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"fecc2fa4-bd98-4c75-8ef6-26edf27e0bdd"} {"level":"info","ts":"2026-07-01T18:10:11.022Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"fecc2fa4-bd98-4c75-8ef6-26edf27e0bdd","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""} {"level":"info","ts":"2026-07-01T18:10:11.022Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"fecc2fa4-bd98-4c75-8ef6-26edf27e0bdd","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"unknown-version-rayjob-fgrbl","rayClusterName":"unknown-version-rayjob-4psvl","jobDeploymentStatus":"Initializing","startTime":"2026-07-01T18:10:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}} {"level":"info","ts":"2026-07-01T18:10:11.022Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"fecc2fa4-bd98-4c75-8ef6-26edf27e0bdd","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"} {"level":"info","ts":"2026-07-01T18:10:11.036Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"e1ff5ac5-cd72-420d-aaea-64f7cdac34c6","JobStatus":"","JobDeploymentStatus":"","SubmissionMode":"K8sJobMode"} {"level":"info","ts":"2026-07-01T18:10:11.036Z","logger":"controllers.RayJob","msg":"JobDeploymentStatusNew","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"e1ff5ac5-cd72-420d-aaea-64f7cdac34c6"} {"level":"info","ts":"2026-07-01T18:10:11.036Z","logger":"controllers.RayJob","msg":"initRayJobStatusIfNeed","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"e1ff5ac5-cd72-420d-aaea-64f7cdac34c6","shouldUpdateStatus":true,"jobId":"","rayClusterName":"","jobStatus":""} {"level":"info","ts":"2026-07-01T18:10:11.036Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"e1ff5ac5-cd72-420d-aaea-64f7cdac34c6","oldRayJobStatus":{"rayJobInfo":{},"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}},"newRayJobStatus":{"rayJobInfo":{},"jobId":"unknown-version-rayjob-sqfxz","rayClusterName":"unknown-version-rayjob-vbnt5","jobDeploymentStatus":"Initializing","startTime":"2026-07-01T18:10:11Z","succeeded":0,"failed":0,"rayClusterStatus":{"desiredCPU":"0","desiredMemory":"0","desiredGPU":"0","desiredTPU":"0","head":{}}}} {"level":"info","ts":"2026-07-01T18:10:11.036Z","logger":"controllers.RayJob","msg":"updateRayJobStatus","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"e1ff5ac5-cd72-420d-aaea-64f7cdac34c6","old JobStatus":"","new JobStatus":"","old JobDeploymentStatus":"","new JobDeploymentStatus":"Initializing"} {"level":"info","ts":"2026-07-01T18:10:11.048Z","logger":"controllers.RayJob","msg":"Failed to update RayJob status","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"e1ff5ac5-cd72-420d-aaea-64f7cdac34c6","error":"Operation cannot be fulfilled on rayjobs.ray.io \"unknown-version-rayjob\": the object has been modified; please apply your changes to the latest version and try again"} {"level":"info","ts":"2026-07-01T18:10:11.048Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"e1ff5ac5-cd72-420d-aaea-64f7cdac34c6"} {"level":"error","ts":"2026-07-01T18:10:11.048Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"e1ff5ac5-cd72-420d-aaea-64f7cdac34c6","error":"Operation cannot be fulfilled on rayjobs.ray.io \"unknown-version-rayjob\": the object has been modified; please apply your changes to the latest version and try again","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:11.049Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"203110e2-eb48-428b-a8a4-7c8ffa024ee0","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"} {"level":"info","ts":"2026-07-01T18:10:11.049Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"203110e2-eb48-428b-a8a4-7c8ffa024ee0","name":"unknown-version-rayjob-4psvl"} {"level":"info","ts":"2026-07-01T18:10:11.049Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"203110e2-eb48-428b-a8a4-7c8ffa024ee0","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.049Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"203110e2-eb48-428b-a8a4-7c8ffa024ee0","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.052Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"203110e2-eb48-428b-a8a4-7c8ffa024ee0"} {"level":"error","ts":"2026-07-01T18:10:11.052Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"203110e2-eb48-428b-a8a4-7c8ffa024ee0","error":"rayclusters.ray.io \"unknown-version-rayjob-4psvl\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:11.054Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"5eb5a924-3229-420f-ad41-93ba7f638ae5","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"} {"level":"info","ts":"2026-07-01T18:10:11.054Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"5eb5a924-3229-420f-ad41-93ba7f638ae5","name":"unknown-version-rayjob-4psvl"} {"level":"info","ts":"2026-07-01T18:10:11.054Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"5eb5a924-3229-420f-ad41-93ba7f638ae5","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.054Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"5eb5a924-3229-420f-ad41-93ba7f638ae5","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.056Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"5eb5a924-3229-420f-ad41-93ba7f638ae5"} {"level":"error","ts":"2026-07-01T18:10:11.056Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"5eb5a924-3229-420f-ad41-93ba7f638ae5","error":"rayclusters.ray.io \"unknown-version-rayjob-4psvl\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:11.077Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"73b386c5-ee23-4e93-a032-f0566182f6cd","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"} {"level":"info","ts":"2026-07-01T18:10:11.077Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"73b386c5-ee23-4e93-a032-f0566182f6cd","name":"unknown-version-rayjob-4psvl"} {"level":"info","ts":"2026-07-01T18:10:11.077Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"73b386c5-ee23-4e93-a032-f0566182f6cd","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.077Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"73b386c5-ee23-4e93-a032-f0566182f6cd","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.080Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"73b386c5-ee23-4e93-a032-f0566182f6cd"} {"level":"error","ts":"2026-07-01T18:10:11.080Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"73b386c5-ee23-4e93-a032-f0566182f6cd","error":"rayclusters.ray.io \"unknown-version-rayjob-4psvl\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:11.121Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"c6f22f51-e138-4c13-8e41-291066e9eecc","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"} {"level":"info","ts":"2026-07-01T18:10:11.121Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"c6f22f51-e138-4c13-8e41-291066e9eecc","name":"unknown-version-rayjob-4psvl"} {"level":"info","ts":"2026-07-01T18:10:11.121Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"c6f22f51-e138-4c13-8e41-291066e9eecc","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.121Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"c6f22f51-e138-4c13-8e41-291066e9eecc","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.124Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"c6f22f51-e138-4c13-8e41-291066e9eecc"} {"level":"error","ts":"2026-07-01T18:10:11.124Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"c6f22f51-e138-4c13-8e41-291066e9eecc","error":"rayclusters.ray.io \"unknown-version-rayjob-4psvl\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:11.204Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"04c49e9e-0143-433a-bd22-bd444d3739f3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"} {"level":"info","ts":"2026-07-01T18:10:11.204Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"04c49e9e-0143-433a-bd22-bd444d3739f3","name":"unknown-version-rayjob-4psvl"} {"level":"info","ts":"2026-07-01T18:10:11.204Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"04c49e9e-0143-433a-bd22-bd444d3739f3","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.204Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"04c49e9e-0143-433a-bd22-bd444d3739f3","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.207Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"04c49e9e-0143-433a-bd22-bd444d3739f3"} {"level":"error","ts":"2026-07-01T18:10:11.208Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"04c49e9e-0143-433a-bd22-bd444d3739f3","error":"rayclusters.ray.io \"unknown-version-rayjob-4psvl\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:11.368Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"3c23df12-574f-4295-9790-d1e891cc8922","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"} {"level":"info","ts":"2026-07-01T18:10:11.368Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"3c23df12-574f-4295-9790-d1e891cc8922","name":"unknown-version-rayjob-4psvl"} {"level":"info","ts":"2026-07-01T18:10:11.368Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"3c23df12-574f-4295-9790-d1e891cc8922","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.368Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"3c23df12-574f-4295-9790-d1e891cc8922","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.371Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"3c23df12-574f-4295-9790-d1e891cc8922"} {"level":"error","ts":"2026-07-01T18:10:11.371Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"3c23df12-574f-4295-9790-d1e891cc8922","error":"rayclusters.ray.io \"unknown-version-rayjob-4psvl\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:11.693Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"b0d0878f-6866-4399-ac14-bc9d87ec7305","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"} {"level":"info","ts":"2026-07-01T18:10:11.693Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"b0d0878f-6866-4399-ac14-bc9d87ec7305","name":"unknown-version-rayjob-4psvl"} {"level":"info","ts":"2026-07-01T18:10:11.693Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"b0d0878f-6866-4399-ac14-bc9d87ec7305","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.693Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"b0d0878f-6866-4399-ac14-bc9d87ec7305","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:11.696Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"b0d0878f-6866-4399-ac14-bc9d87ec7305"} {"level":"error","ts":"2026-07-01T18:10:11.696Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"b0d0878f-6866-4399-ac14-bc9d87ec7305","error":"rayclusters.ray.io \"unknown-version-rayjob-4psvl\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:12.337Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"37720f39-f1f3-4821-802f-9ac45a9323a3","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"} {"level":"info","ts":"2026-07-01T18:10:12.337Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"37720f39-f1f3-4821-802f-9ac45a9323a3","name":"unknown-version-rayjob-4psvl"} {"level":"info","ts":"2026-07-01T18:10:12.337Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"37720f39-f1f3-4821-802f-9ac45a9323a3","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:12.337Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"37720f39-f1f3-4821-802f-9ac45a9323a3","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:12.341Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"37720f39-f1f3-4821-802f-9ac45a9323a3"} {"level":"error","ts":"2026-07-01T18:10:12.341Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"37720f39-f1f3-4821-802f-9ac45a9323a3","error":"rayclusters.ray.io \"unknown-version-rayjob-4psvl\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:13.622Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"29e669b4-7972-4027-b509-c38d2f10eb58","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"} {"level":"info","ts":"2026-07-01T18:10:13.622Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"29e669b4-7972-4027-b509-c38d2f10eb58","name":"unknown-version-rayjob-4psvl"} {"level":"info","ts":"2026-07-01T18:10:13.622Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"29e669b4-7972-4027-b509-c38d2f10eb58","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:13.622Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"29e669b4-7972-4027-b509-c38d2f10eb58","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:13.647Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"29e669b4-7972-4027-b509-c38d2f10eb58"} {"level":"error","ts":"2026-07-01T18:10:13.647Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"29e669b4-7972-4027-b509-c38d2f10eb58","error":"rayclusters.ray.io \"unknown-version-rayjob-4psvl\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:16.208Z","logger":"controllers.RayJob","msg":"RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"6d61f30b-3db2-42cc-a4e1-b69942e186ea","JobStatus":"","JobDeploymentStatus":"Initializing","SubmissionMode":"K8sJobMode"} {"level":"info","ts":"2026-07-01T18:10:16.208Z","logger":"controllers.RayJob","msg":"try to find existing RayCluster instance","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"6d61f30b-3db2-42cc-a4e1-b69942e186ea","name":"unknown-version-rayjob-4psvl"} {"level":"info","ts":"2026-07-01T18:10:16.208Z","logger":"controllers.RayJob","msg":"RayCluster not found","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"6d61f30b-3db2-42cc-a4e1-b69942e186ea","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:16.208Z","logger":"controllers.RayJob","msg":"RayCluster not found, creating RayCluster!","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"6d61f30b-3db2-42cc-a4e1-b69942e186ea","RayCluster":{"name":"unknown-version-rayjob-4psvl","namespace":"test-ns-taafe"}} {"level":"info","ts":"2026-07-01T18:10:16.212Z","logger":"controllers.RayJob","msg":"Warning: Reconciler returned both a non-zero result and a non-nil error. The result will always be ignored if the error is non-nil and the non-nil error causes requeuing with exponential backoff. For more details, see: https://pkg.go.dev/sigs.k8s.io/controller-runtime/pkg/reconcile#Reconciler","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"6d61f30b-3db2-42cc-a4e1-b69942e186ea"} {"level":"error","ts":"2026-07-01T18:10:16.212Z","logger":"controllers.RayJob","msg":"Reconciler error","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"6d61f30b-3db2-42cc-a4e1-b69942e186ea","error":"rayclusters.ray.io \"unknown-version-rayjob-4psvl\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated","stacktrace":"sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:474\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:18.121Z","logger":"controllers.RayJob","msg":"RayJob is being deleted","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"868119fe-34ec-4a47-9e82-9de7f9a39ce9","DeletionTimestamp":"2026-07-01 18:10:18 +0000 UTC"} {"level":"error","ts":"2026-07-01T18:10:18.121Z","logger":"controllers.RayJob","msg":"Failed to get RayCluster","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"868119fe-34ec-4a47-9e82-9de7f9a39ce9","error":"RayCluster.ray.io \"unknown-version-rayjob-4psvl\" not found","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:115\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:18.121Z","logger":"controllers.RayJob","msg":"Stop a ray job","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"868119fe-34ec-4a47-9e82-9de7f9a39ce9","rayJob":"unknown-version-rayjob-fgrbl"} {"level":"error","ts":"2026-07-01T18:10:18.121Z","logger":"controllers.RayJob","msg":"Failed to stop job for RayJob","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"868119fe-34ec-4a47-9e82-9de7f9a39ce9","error":"Post \"http:///api/jobs/unknown-version-rayjob-fgrbl/stop\": http: no Host in request URL","stacktrace":"github.com/ray-project/kuberay/ray-operator/controllers/ray.(*RayJobReconciler).Reconcile\n\t/workspace/controllers/ray/rayjob_controller.go:123\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Reconcile\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:216\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:461\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:421\nsigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func1.1\n\t/cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.22.1/pkg/internal/controller/controller.go:296"} {"level":"info","ts":"2026-07-01T18:10:18.122Z","logger":"controllers.RayJob","msg":"Remove the finalizer no matter StopJob() succeeds or not.","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"868119fe-34ec-4a47-9e82-9de7f9a39ce9","finalizer":"ray.io/rayjob-finalizer"} {"level":"info","ts":"2026-07-01T18:10:18.136Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"82288359-4318-4ec4-a77e-505df874b4e9"} {"level":"info","ts":"2026-07-01T18:10:21.137Z","logger":"controllers.RayJob","msg":"RayJob resource not found. Ignoring since object must be deleted","RayJob":{"name":"unknown-version-rayjob","namespace":"test-ns-taafe"},"reconcileID":"4e2fb0a4-9cf5-4fa1-b19e-cf991836245a"} ==================== WORKLOAD POD LOG: redhat-ods-applications/llmisvc-controller-manager-586c85c465-64pgz.log ==================== ==================== WORKLOAD POD LOG: redhat-ods-applications/maas-api-84c4b8dcb6-8mf4j.log ==================== [GIN] 2026/07/01 - 18:04:21 | 200 | 30.518µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:04:22 | 200 | 31.45µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:04:26 | 200 | 28.705µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:04:31 | 200 | 30.997µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:04:32 | 200 | 31.929µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:04:36 | 200 | 31.476µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:04:41 | 200 | 35.35µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:04:42 | 200 | 32.653µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:04:46 | 200 | 33.892µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:04:51 | 200 | 41.446µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:04:52 | 200 | 36.869µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:04:56 | 200 | 32.704µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:01 | 200 | 31.648µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:02 | 200 | 34.241µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:06 | 200 | 33.816µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:11 | 200 | 32.869µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:12 | 200 | 33.901µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:16 | 200 | 31.007µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:21 | 200 | 32.437µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:22 | 200 | 30.017µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:26 | 200 | 51.376µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:31 | 200 | 35.209µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:32 | 200 | 32.669µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:36 | 200 | 46.62µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:41 | 200 | 36.628µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:42 | 200 | 44.107µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:46 | 200 | 44.001µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:51 | 200 | 34.082µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:52 | 200 | 32.345µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:05:56 | 200 | 30.41µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:01 | 200 | 32.282µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:02 | 200 | 48.356µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:06 | 200 | 40.835µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:11 | 200 | 44.277µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:12 | 200 | 35.294µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:16 | 200 | 31.266µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:21 | 200 | 30.129µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:22 | 200 | 43.235µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:26 | 200 | 40.193µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:31 | 200 | 38.448µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:32 | 200 | 33.25µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:36 | 200 | 44.306µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:41 | 200 | 38.896µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:42 | 200 | 31.368µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:46 | 200 | 31.401µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:51 | 200 | 32.593µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:52 | 200 | 51.006µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:06:56 | 200 | 30.75µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:01 | 200 | 57.231µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:02 | 200 | 105.217µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:06 | 200 | 45.843µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:11 | 200 | 35.315µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:12 | 200 | 31.352µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:16 | 200 | 33.408µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:21 | 200 | 34.389µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:22 | 200 | 35.56µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:26 | 200 | 57.611µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:31 | 200 | 44.928µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:32 | 200 | 42.392µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:36 | 200 | 45.017µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:41 | 200 | 29.914µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:42 | 200 | 69.284µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:46 | 200 | 34.292µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:51 | 200 | 30.934µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:52 | 200 | 33.509µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:07:56 | 200 | 30.711µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:01 | 200 | 41.551µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:02 | 200 | 77.495µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:06 | 200 | 56.747µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:11 | 200 | 29.412µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:12 | 200 | 40.167µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:16 | 200 | 52.994µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:21 | 200 | 34.868µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:22 | 200 | 42.961µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:26 | 200 | 31.402µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:31 | 200 | 35.304µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:32 | 200 | 32.383µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:36 | 200 | 32.113µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:41 | 200 | 33.527µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:42 | 200 | 31.605µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:46 | 200 | 57.364µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:51 | 200 | 32.377µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:52 | 200 | 36.908µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:08:56 | 200 | 32.21µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:01 | 200 | 43.141µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:02 | 200 | 33.669µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:06 | 200 | 31.253µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:11 | 200 | 55.15µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:12 | 200 | 57.007µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:16 | 200 | 33.285µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:21 | 200 | 50.159µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:22 | 200 | 38.308µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:26 | 200 | 32.306µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:31 | 200 | 30.524µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:32 | 200 | 47.777µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:36 | 200 | 43.228µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:41 | 200 | 32.002µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:42 | 200 | 32.765µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:46 | 200 | 59.649µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:51 | 200 | 34.388µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:52 | 200 | 62.228µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:09:56 | 200 | 33.135µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:01 | 200 | 40.617µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:02 | 200 | 47.516µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:06 | 200 | 44.693µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:11 | 200 | 41.119µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:12 | 200 | 35.272µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:16 | 200 | 74.097µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:21 | 200 | 31.606µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:22 | 200 | 41.792µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:26 | 200 | 42.631µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:31 | 200 | 33.656µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:32 | 200 | 31.862µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:36 | 200 | 44.376µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:41 | 200 | 47.451µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:42 | 200 | 32.757µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:46 | 200 | 29.904µs | 10.131.0.2 | GET "/health" [GIN] 2026/07/01 - 18:10:51 | 200 | 42.812µs | 10.131.0.2 | GET "/health" ==================== WORKLOAD POD LOG: redhat-ods-applications/maas-api-key-cleanup-29715450-7zsmn.log ==================== ==================== WORKLOAD POD LOG: redhat-ods-applications/maas-api-key-cleanup-29715465-l5bpz.log ==================== ==================== WORKLOAD POD LOG: redhat-ods-applications/maas-api-key-cleanup-29715480-xdsht.log ==================== ==================== WORKLOAD POD LOG: redhat-ods-applications/maas-controller-656b6867cb-x8bsw.log ==================== {"level":"error","ts":"2026-07-01T18:09:50Z","logger":"setup","msg":"unable to ensure subscription namespace exists","namespace":"models-as-a-service","error":"unable to check if namespace \"models-as-a-service\" exists: Unauthorized","stacktrace":"main.main\n\tgithub.com/opendatahub-io/models-as-a-service/maas-controller/cmd/manager/main.go:466\nruntime.main\n\truntime/proc.go:290"} ==================== WORKLOAD POD LOG: redhat-ods-applications/maas-postgres-d66f546cf-vsktb.log ==================== ==================== WORKLOAD POD LOG: redhat-ods-applications/model-serving-api-78bbbb9bf8-gz7fv.log ==================== ==================== WORKLOAD POD LOG: redhat-ods-applications/odh-model-controller-54bf759f64-hr25g.log ==================== 2026-07-01T18:10:10Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-trusted-ca-bundle", "reconcileID": "d174c02c-7b06-44b2-9c8a-8bdf350d8db8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-r4phu", "create": "odh-kserve-custom-ca-bundle"} 2026-07-01T18:10:10Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-trusted-ca-bundle", "reconcileID": "d174c02c-7b06-44b2-9c8a-8bdf350d8db8", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a3e55bc3-d9d9-4cbe-88d7-f35cb9a506a5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "openshift-service-ca.crt", "reconcileID": "86529032-f71b-4e51-8595-8cf0add59c6e", "ConfigMap": "openshift-service-ca.crt", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:10Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-trusted-ca-bundle", "reconcileID": "6e8b95f1-6459-42fc-a675-3d3c543171c4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-r4phu", "update": "odh-kserve-custom-ca-bundle"} 2026-07-01T18:10:10Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-trusted-ca-bundle", "reconcileID": "6e8b95f1-6459-42fc-a675-3d3c543171c4", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:10Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "openshift-service-ca.crt", "reconcileID": "0b89d612-9212-469d-b995-1eb8f9933b86", "ConfigMap": "openshift-service-ca.crt", "namespace": "test-ns-r4phu", "update": "odh-kserve-custom-ca-bundle"} 2026-07-01T18:10:10Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "openshift-service-ca.crt", "reconcileID": "0b89d612-9212-469d-b995-1eb8f9933b86", "ConfigMap": "openshift-service-ca.crt", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "f5c640e3-e827-4e30-86df-76509cbfcb40", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "openshift-service-ca.crt", "reconcileID": "dd149049-c0ad-4ef4-9259-d9f1833b5d9a", "ConfigMap": "openshift-service-ca.crt", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "openshift-service-ca.crt", "reconcileID": "08d66b82-4bb4-4042-98af-9772a49d92a7", "ConfigMap": "openshift-service-ca.crt", "namespace": "test-ns-taafe"} 2026-07-01T18:10:10Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "openshift-service-ca.crt", "reconcileID": "940359bb-7cb6-491e-811a-a78071f1a839", "ConfigMap": "openshift-service-ca.crt", "namespace": "test-ns-taafe", "create": "odh-kserve-custom-ca-bundle"} 2026-07-01T18:10:10Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "openshift-service-ca.crt", "reconcileID": "940359bb-7cb6-491e-811a-a78071f1a839", "ConfigMap": "openshift-service-ca.crt", "namespace": "test-ns-taafe"} 2026-07-01T18:10:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-trusted-ca-bundle", "reconcileID": "9e5512d9-ff32-4da7-8940-ec4483d4053b", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-taafe"} 2026-07-01T18:10:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "c64d8fd2-5446-474b-855b-1bae7252a695", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-taafe"} 2026-07-01T18:10:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "openshift-service-ca.crt", "reconcileID": "68894e72-faa5-4d25-ad7f-c7bcdff8983e", "ConfigMap": "openshift-service-ca.crt", "namespace": "test-ns-taafe"} 2026-07-01T18:10:10Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-trusted-ca-bundle", "reconcileID": "ab306464-14d6-40df-a7e7-2cb0198ed6b9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-taafe", "update": "odh-kserve-custom-ca-bundle"} 2026-07-01T18:10:10Z DEBUG Deleted the storage-config Secret to update the custom cert {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-trusted-ca-bundle", "reconcileID": "ab306464-14d6-40df-a7e7-2cb0198ed6b9", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-taafe"} 2026-07-01T18:10:10Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "44fb05d0-822c-460f-a8b1-5630aabe5fb2", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-taafe"} 2026-07-01T18:10:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8bac3d62-32b7-4c58-99ba-bc2baf8af92a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-taafe"} 2026-07-01T18:10:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8bac3d62-32b7-4c58-99ba-bc2baf8af92a", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-taafe", "create": "odh-kserve-custom-ca-bundle"} 2026-07-01T18:10:16Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "8bac3d62-32b7-4c58-99ba-bc2baf8af92a", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-01T18:10:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-trusted-ca-bundle", "reconcileID": "35d0543e-16fa-4adb-907a-53f14f89b817", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-taafe"} 2026-07-01T18:10:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-trusted-ca-bundle", "reconcileID": "35d0543e-16fa-4adb-907a-53f14f89b817", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-taafe", "create": "odh-kserve-custom-ca-bundle"} 2026-07-01T18:10:16Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-trusted-ca-bundle", "reconcileID": "35d0543e-16fa-4adb-907a-53f14f89b817", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-01T18:10:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a79381b5-e7e1-4280-a4cb-7eb0e2c01f2e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-taafe"} 2026-07-01T18:10:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a79381b5-e7e1-4280-a4cb-7eb0e2c01f2e", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-taafe", "create": "odh-kserve-custom-ca-bundle"} 2026-07-01T18:10:16Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "a79381b5-e7e1-4280-a4cb-7eb0e2c01f2e", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace test-ns-taafe because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-01T18:10:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "openshift-service-ca.crt", "reconcileID": "a9b0e562-3124-4d29-9745-a1b03f278d18", "ConfigMap": "openshift-service-ca.crt", "namespace": "test-ns-taafe"} 2026-07-01T18:10:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "openshift-service-ca.crt", "reconcileID": "a9b0e562-3124-4d29-9745-a1b03f278d18", "ConfigMap": "openshift-service-ca.crt", "namespace": "test-ns-taafe"} 2026-07-01T18:10:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-trusted-ca-bundle", "reconcileID": "6b426dd0-6a9f-4cd9-9444-65e8e426e7ca", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-taafe"} 2026-07-01T18:10:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-trusted-ca-bundle", "reconcileID": "6b426dd0-6a9f-4cd9-9444-65e8e426e7ca", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-taafe"} 2026-07-01T18:10:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0fd3c592-b4ae-477e-8004-473bb51a7715", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-taafe"} 2026-07-01T18:10:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-taafe"}, "namespace": "test-ns-taafe", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "0fd3c592-b4ae-477e-8004-473bb51a7715", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-taafe"} 2026-07-01T18:10:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3b43847c-e127-4643-ad10-d718aec7d8f5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3b43847c-e127-4643-ad10-d718aec7d8f5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-r4phu", "create": "odh-kserve-custom-ca-bundle"} 2026-07-01T18:10:16Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "3b43847c-e127-4643-ad10-d718aec7d8f5", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace test-ns-r4phu because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-01T18:10:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-trusted-ca-bundle", "reconcileID": "1e6bafb6-837e-437e-aa08-ac8424dc9553", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-trusted-ca-bundle", "reconcileID": "1e6bafb6-837e-437e-aa08-ac8424dc9553", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-r4phu", "create": "odh-kserve-custom-ca-bundle"} 2026-07-01T18:10:16Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-trusted-ca-bundle", "reconcileID": "1e6bafb6-837e-437e-aa08-ac8424dc9553", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace test-ns-r4phu because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-01T18:10:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "79af2960-cf59-4b0f-8386-f0abddb6c2ae", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:16Z DEBUG Delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "79af2960-cf59-4b0f-8386-f0abddb6c2ae", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-r4phu", "create": "odh-kserve-custom-ca-bundle"} 2026-07-01T18:10:16Z ERROR Reconciler error {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "79af2960-cf59-4b0f-8386-f0abddb6c2ae", "error": "configmaps \"odh-kserve-custom-ca-bundle\" is forbidden: unable to create new content in namespace test-ns-r4phu because it is being terminated"} sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).reconcileHandler /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:316 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).processNextWorkItem /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:263 sigs.k8s.io/controller-runtime/pkg/internal/controller.(*Controller[...]).Start.func2.2 /cachi2/output/deps/gomod/pkg/mod/sigs.k8s.io/controller-runtime@v0.19.7/pkg/internal/controller/controller.go:224 2026-07-01T18:10:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "openshift-service-ca.crt", "reconcileID": "a3f4a583-f23b-48a2-ab4d-cde1a153183b", "ConfigMap": "openshift-service-ca.crt", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"openshift-service-ca.crt","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "openshift-service-ca.crt", "reconcileID": "a3f4a583-f23b-48a2-ab4d-cde1a153183b", "ConfigMap": "openshift-service-ca.crt", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-trusted-ca-bundle", "reconcileID": "8b65b8fe-6cf9-4d5b-854d-13f8cd59a263", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-trusted-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-trusted-ca-bundle", "reconcileID": "8b65b8fe-6cf9-4d5b-854d-13f8cd59a263", "ConfigMap": "odh-trusted-ca-bundle", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:16Z INFO Cert ConfigMap not found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e054f409-b98e-4708-a731-739fd0900aa5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-r4phu"} 2026-07-01T18:10:16Z DEBUG No delta found {"controller": "core-configmap", "controllerGroup": "", "controllerKind": "ConfigMap", "ConfigMap": {"name":"odh-kserve-custom-ca-bundle","namespace":"test-ns-r4phu"}, "namespace": "test-ns-r4phu", "name": "odh-kserve-custom-ca-bundle", "reconcileID": "e054f409-b98e-4708-a731-739fd0900aa5", "ConfigMap": "odh-kserve-custom-ca-bundle", "namespace": "test-ns-r4phu"} ==================== WORKLOAD POD LOG: redhat-ods-applications/postgres-595b8d5f6-smmgf.log ==================== ==================== WORKLOAD POD LOG: redhat-ods-applications/rhods-dashboard-6b5cc9785c-4z7kp.log ==================== time=2026-07-01T18:04:36.672Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" time=2026-07-01T18:05:06.667Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" time=2026-07-01T18:05:36.668Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" time=2026-07-01T18:06:06.669Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" time=2026-07-01T18:06:36.674Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" time=2026-07-01T18:07:06.669Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" time=2026-07-01T18:07:36.668Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" time=2026-07-01T18:08:06.668Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" time=2026-07-01T18:08:36.668Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" time=2026-07-01T18:09:06.670Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" time=2026-07-01T18:09:36.668Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" time=2026-07-01T18:10:06.669Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" time=2026-07-01T18:10:36.667Z level=WARN msg="MLflow CR auto-discovery failed, MLflow endpoints will return 503" error="no MLflow CR found in the cluster" {"level":30,"time":1782929066257,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-f","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":40356},"msg":"incoming request"} {"level":30,"time":1782929066257,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-f","res":{"statusCode":200},"responseTime":0.43300867080688477,"msg":"request completed"} {"level":30,"time":1782929087674,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-g","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":55068},"msg":"incoming request"} {"level":30,"time":1782929087674,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-g","res":{"statusCode":200},"responseTime":0.4516291618347168,"msg":"request completed"} {"level":30,"time":1782929096254,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-h","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":55140},"msg":"incoming request"} {"level":30,"time":1782929096254,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-h","res":{"statusCode":200},"responseTime":0.5845861434936523,"msg":"request completed"} {"level":30,"time":1782929117672,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-i","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":57680},"msg":"incoming request"} {"level":30,"time":1782929117672,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-i","res":{"statusCode":200},"responseTime":0.4228200912475586,"msg":"request completed"} {"level":30,"time":1782929126255,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-j","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":57758},"msg":"incoming request"} {"level":30,"time":1782929126256,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-j","res":{"statusCode":200},"responseTime":0.6426687240600586,"msg":"request completed"} {"level":30,"time":1782929147672,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-k","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":57198},"msg":"incoming request"} {"level":30,"time":1782929147672,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-k","res":{"statusCode":200},"responseTime":0.40558910369873047,"msg":"request completed"} {"level":30,"time":1782929156257,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-l","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":57254},"msg":"incoming request"} {"level":30,"time":1782929156258,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-l","res":{"statusCode":200},"responseTime":0.8346438407897949,"msg":"request completed"} {"level":30,"time":1782929177673,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-m","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":58500},"msg":"incoming request"} {"level":30,"time":1782929177673,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-m","res":{"statusCode":200},"responseTime":0.4265751838684082,"msg":"request completed"} {"level":30,"time":1782929186256,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-n","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":58590},"msg":"incoming request"} {"level":30,"time":1782929186256,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-n","res":{"statusCode":200},"responseTime":0.525507926940918,"msg":"request completed"} {"level":30,"time":1782929207674,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-o","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":39364},"msg":"incoming request"} {"level":30,"time":1782929207675,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-o","res":{"statusCode":200},"responseTime":0.5022611618041992,"msg":"request completed"} {"level":30,"time":1782929216257,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-p","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":39436},"msg":"incoming request"} {"level":30,"time":1782929216258,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-p","res":{"statusCode":200},"responseTime":0.49576902389526367,"msg":"request completed"} {"level":30,"time":1782929237672,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-q","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":51706},"msg":"incoming request"} {"level":30,"time":1782929237673,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-q","res":{"statusCode":200},"responseTime":0.43933677673339844,"msg":"request completed"} {"level":30,"time":1782929246254,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-r","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":51760},"msg":"incoming request"} {"level":30,"time":1782929246256,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-r","res":{"statusCode":200},"responseTime":1.3453450202941895,"msg":"request completed"} {"level":30,"time":1782929267674,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-s","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":37974},"msg":"incoming request"} {"level":30,"time":1782929267674,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-s","res":{"statusCode":200},"responseTime":0.3784008026123047,"msg":"request completed"} {"level":30,"time":1782929276255,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-t","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":38072},"msg":"incoming request"} {"level":30,"time":1782929276256,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-t","res":{"statusCode":200},"responseTime":0.5900206565856934,"msg":"request completed"} {"level":30,"time":1782929297675,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-u","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":59274},"msg":"incoming request"} {"level":30,"time":1782929297676,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-u","res":{"statusCode":200},"responseTime":0.4798393249511719,"msg":"request completed"} {"level":30,"time":1782929306254,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-v","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":59338},"msg":"incoming request"} {"level":30,"time":1782929306255,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-v","res":{"statusCode":200},"responseTime":0.5234827995300293,"msg":"request completed"} {"level":30,"time":1782929327674,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-w","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":47794},"msg":"incoming request"} {"level":30,"time":1782929327674,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-w","res":{"statusCode":200},"responseTime":0.37909507751464844,"msg":"request completed"} {"level":30,"time":1782929336256,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-x","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":47876},"msg":"incoming request"} {"level":30,"time":1782929336257,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-x","res":{"statusCode":200},"responseTime":0.5244278907775879,"msg":"request completed"} {"level":30,"time":1782929357674,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-y","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":45334},"msg":"incoming request"} {"level":30,"time":1782929357676,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-y","res":{"statusCode":200},"responseTime":1.0558662414550781,"msg":"request completed"} {"level":30,"time":1782929366257,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-z","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":45408},"msg":"incoming request"} {"level":30,"time":1782929366257,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-z","res":{"statusCode":200},"responseTime":0.4734230041503906,"msg":"request completed"} {"level":30,"time":1782929387672,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-10","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":49782},"msg":"incoming request"} {"level":30,"time":1782929387672,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-10","res":{"statusCode":200},"responseTime":0.44593286514282227,"msg":"request completed"} {"level":30,"time":1782929396256,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-11","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":49894},"msg":"incoming request"} {"level":30,"time":1782929396256,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-11","res":{"statusCode":200},"responseTime":0.42231082916259766,"msg":"request completed"} {"level":30,"time":1782929417673,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-12","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":39242},"msg":"incoming request"} {"level":30,"time":1782929417673,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-12","res":{"statusCode":200},"responseTime":0.4064960479736328,"msg":"request completed"} {"level":30,"time":1782929426256,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-13","req":{"method":"GET","url":"/","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":39338},"msg":"incoming request"} {"level":30,"time":1782929426257,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-13","res":{"statusCode":200},"responseTime":0.44747400283813477,"msg":"request completed"} {"level":30,"time":1782929447672,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-14","req":{"method":"GET","url":"/api/health","hostname":"127.0.0.1:8080","remoteAddress":"127.0.0.1","remotePort":36572},"msg":"incoming request"} {"level":30,"time":1782929447672,"pid":12,"hostname":"rhods-dashboard-6b5cc9785c-4z7kp","reqId":"req-14","res":{"statusCode":200},"responseTime":0.39040374755859375,"msg":"request completed"}